|
@@ -53,16 +53,6 @@
|
|
#define CREATE_TRACE_POINTS
|
|
#define CREATE_TRACE_POINTS
|
|
#include <trace/events/vmscan.h>
|
|
#include <trace/events/vmscan.h>
|
|
|
|
|
|
-/*
|
|
|
|
- * reclaim_mode determines how the inactive list is shrunk
|
|
|
|
- * RECLAIM_MODE_SINGLE: Reclaim only order-0 pages
|
|
|
|
- * RECLAIM_MODE_COMPACTION: For high-order allocations, reclaim a number of
|
|
|
|
- * order-0 pages and then compact the zone
|
|
|
|
- */
|
|
|
|
-typedef unsigned __bitwise__ reclaim_mode_t;
|
|
|
|
-#define RECLAIM_MODE_SINGLE ((__force reclaim_mode_t)0x01u)
|
|
|
|
-#define RECLAIM_MODE_COMPACTION ((__force reclaim_mode_t)0x10u)
|
|
|
|
-
|
|
|
|
struct scan_control {
|
|
struct scan_control {
|
|
/* Incremented by the number of inactive pages that were scanned */
|
|
/* Incremented by the number of inactive pages that were scanned */
|
|
unsigned long nr_scanned;
|
|
unsigned long nr_scanned;
|
|
@@ -88,12 +78,6 @@ struct scan_control {
|
|
|
|
|
|
int order;
|
|
int order;
|
|
|
|
|
|
- /*
|
|
|
|
- * Intend to reclaim enough continuous memory rather than reclaim
|
|
|
|
- * enough amount of memory. i.e, mode for high order allocation.
|
|
|
|
- */
|
|
|
|
- reclaim_mode_t reclaim_mode;
|
|
|
|
-
|
|
|
|
/*
|
|
/*
|
|
* The memory cgroup that hit its limit and as a result is the
|
|
* The memory cgroup that hit its limit and as a result is the
|
|
* primary target of this reclaim invocation.
|
|
* primary target of this reclaim invocation.
|
|
@@ -356,25 +340,6 @@ out:
|
|
return ret;
|
|
return ret;
|
|
}
|
|
}
|
|
|
|
|
|
-static void set_reclaim_mode(int priority, struct scan_control *sc)
|
|
|
|
-{
|
|
|
|
- /*
|
|
|
|
- * Restrict reclaim/compaction to costly allocations or when
|
|
|
|
- * under memory pressure
|
|
|
|
- */
|
|
|
|
- if (COMPACTION_BUILD && sc->order &&
|
|
|
|
- (sc->order > PAGE_ALLOC_COSTLY_ORDER ||
|
|
|
|
- priority < DEF_PRIORITY - 2))
|
|
|
|
- sc->reclaim_mode = RECLAIM_MODE_COMPACTION;
|
|
|
|
- else
|
|
|
|
- sc->reclaim_mode = RECLAIM_MODE_SINGLE;
|
|
|
|
-}
|
|
|
|
-
|
|
|
|
-static void reset_reclaim_mode(struct scan_control *sc)
|
|
|
|
-{
|
|
|
|
- sc->reclaim_mode = RECLAIM_MODE_SINGLE;
|
|
|
|
-}
|
|
|
|
-
|
|
|
|
static inline int is_page_cache_freeable(struct page *page)
|
|
static inline int is_page_cache_freeable(struct page *page)
|
|
{
|
|
{
|
|
/*
|
|
/*
|
|
@@ -497,8 +462,7 @@ static pageout_t pageout(struct page *page, struct address_space *mapping,
|
|
/* synchronous write or broken a_ops? */
|
|
/* synchronous write or broken a_ops? */
|
|
ClearPageReclaim(page);
|
|
ClearPageReclaim(page);
|
|
}
|
|
}
|
|
- trace_mm_vmscan_writepage(page,
|
|
|
|
- trace_reclaim_flags(page, sc->reclaim_mode));
|
|
|
|
|
|
+ trace_mm_vmscan_writepage(page, trace_reclaim_flags(page));
|
|
inc_zone_page_state(page, NR_VMSCAN_WRITE);
|
|
inc_zone_page_state(page, NR_VMSCAN_WRITE);
|
|
return PAGE_SUCCESS;
|
|
return PAGE_SUCCESS;
|
|
}
|
|
}
|
|
@@ -953,7 +917,6 @@ cull_mlocked:
|
|
try_to_free_swap(page);
|
|
try_to_free_swap(page);
|
|
unlock_page(page);
|
|
unlock_page(page);
|
|
putback_lru_page(page);
|
|
putback_lru_page(page);
|
|
- reset_reclaim_mode(sc);
|
|
|
|
continue;
|
|
continue;
|
|
|
|
|
|
activate_locked:
|
|
activate_locked:
|
|
@@ -1348,8 +1311,6 @@ shrink_inactive_list(unsigned long nr_to_scan, struct mem_cgroup_zone *mz,
|
|
return SWAP_CLUSTER_MAX;
|
|
return SWAP_CLUSTER_MAX;
|
|
}
|
|
}
|
|
|
|
|
|
- set_reclaim_mode(priority, sc);
|
|
|
|
-
|
|
|
|
lru_add_drain();
|
|
lru_add_drain();
|
|
|
|
|
|
if (!sc->may_unmap)
|
|
if (!sc->may_unmap)
|
|
@@ -1433,7 +1394,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct mem_cgroup_zone *mz,
|
|
zone_idx(zone),
|
|
zone_idx(zone),
|
|
nr_scanned, nr_reclaimed,
|
|
nr_scanned, nr_reclaimed,
|
|
priority,
|
|
priority,
|
|
- trace_shrink_flags(file, sc->reclaim_mode));
|
|
|
|
|
|
+ trace_shrink_flags(file));
|
|
return nr_reclaimed;
|
|
return nr_reclaimed;
|
|
}
|
|
}
|
|
|
|
|
|
@@ -1512,8 +1473,6 @@ static void shrink_active_list(unsigned long nr_to_scan,
|
|
|
|
|
|
lru_add_drain();
|
|
lru_add_drain();
|
|
|
|
|
|
- reset_reclaim_mode(sc);
|
|
|
|
-
|
|
|
|
if (!sc->may_unmap)
|
|
if (!sc->may_unmap)
|
|
isolate_mode |= ISOLATE_UNMAPPED;
|
|
isolate_mode |= ISOLATE_UNMAPPED;
|
|
if (!sc->may_writepage)
|
|
if (!sc->may_writepage)
|
|
@@ -1826,23 +1785,35 @@ out:
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
|
|
+/* Use reclaim/compaction for costly allocs or under memory pressure */
|
|
|
|
+static bool in_reclaim_compaction(int priority, struct scan_control *sc)
|
|
|
|
+{
|
|
|
|
+ if (COMPACTION_BUILD && sc->order &&
|
|
|
|
+ (sc->order > PAGE_ALLOC_COSTLY_ORDER ||
|
|
|
|
+ priority < DEF_PRIORITY - 2))
|
|
|
|
+ return true;
|
|
|
|
+
|
|
|
|
+ return false;
|
|
|
|
+}
|
|
|
|
+
|
|
/*
|
|
/*
|
|
- * Reclaim/compaction depends on a number of pages being freed. To avoid
|
|
|
|
- * disruption to the system, a small number of order-0 pages continue to be
|
|
|
|
- * rotated and reclaimed in the normal fashion. However, by the time we get
|
|
|
|
- * back to the allocator and call try_to_compact_zone(), we ensure that
|
|
|
|
- * there are enough free pages for it to be likely successful
|
|
|
|
|
|
+ * Reclaim/compaction is used for high-order allocation requests. It reclaims
|
|
|
|
+ * order-0 pages before compacting the zone. should_continue_reclaim() returns
|
|
|
|
+ * true if more pages should be reclaimed such that when the page allocator
|
|
|
|
+ * calls try_to_compact_zone() that it will have enough free pages to succeed.
|
|
|
|
+ * It will give up earlier than that if there is difficulty reclaiming pages.
|
|
*/
|
|
*/
|
|
static inline bool should_continue_reclaim(struct mem_cgroup_zone *mz,
|
|
static inline bool should_continue_reclaim(struct mem_cgroup_zone *mz,
|
|
unsigned long nr_reclaimed,
|
|
unsigned long nr_reclaimed,
|
|
unsigned long nr_scanned,
|
|
unsigned long nr_scanned,
|
|
|
|
+ int priority,
|
|
struct scan_control *sc)
|
|
struct scan_control *sc)
|
|
{
|
|
{
|
|
unsigned long pages_for_compaction;
|
|
unsigned long pages_for_compaction;
|
|
unsigned long inactive_lru_pages;
|
|
unsigned long inactive_lru_pages;
|
|
|
|
|
|
/* If not in reclaim/compaction mode, stop */
|
|
/* If not in reclaim/compaction mode, stop */
|
|
- if (!(sc->reclaim_mode & RECLAIM_MODE_COMPACTION))
|
|
|
|
|
|
+ if (!in_reclaim_compaction(priority, sc))
|
|
return false;
|
|
return false;
|
|
|
|
|
|
/* Consider stopping depending on scan and reclaim activity */
|
|
/* Consider stopping depending on scan and reclaim activity */
|
|
@@ -1944,7 +1915,8 @@ restart:
|
|
|
|
|
|
/* reclaim/compaction might need reclaim to continue */
|
|
/* reclaim/compaction might need reclaim to continue */
|
|
if (should_continue_reclaim(mz, nr_reclaimed,
|
|
if (should_continue_reclaim(mz, nr_reclaimed,
|
|
- sc->nr_scanned - nr_scanned, sc))
|
|
|
|
|
|
+ sc->nr_scanned - nr_scanned,
|
|
|
|
+ priority, sc))
|
|
goto restart;
|
|
goto restart;
|
|
|
|
|
|
throttle_vm_writeout(sc->gfp_mask);
|
|
throttle_vm_writeout(sc->gfp_mask);
|