compaction.c 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218
  1. /*
  2. * linux/mm/compaction.c
  3. *
  4. * Memory compaction for the reduction of external fragmentation. Note that
  5. * this heavily depends upon page migration to do all the real heavy
  6. * lifting
  7. *
  8. * Copyright IBM Corp. 2007-2010 Mel Gorman <mel@csn.ul.ie>
  9. */
  10. #include <linux/swap.h>
  11. #include <linux/migrate.h>
  12. #include <linux/compaction.h>
  13. #include <linux/mm_inline.h>
  14. #include <linux/backing-dev.h>
  15. #include <linux/sysctl.h>
  16. #include <linux/sysfs.h>
  17. #include <linux/balloon_compaction.h>
  18. #include <linux/page-isolation.h>
  19. #include "internal.h"
  20. #ifdef CONFIG_COMPACTION
  21. static inline void count_compact_event(enum vm_event_item item)
  22. {
  23. count_vm_event(item);
  24. }
  25. static inline void count_compact_events(enum vm_event_item item, long delta)
  26. {
  27. count_vm_events(item, delta);
  28. }
  29. #else
  30. #define count_compact_event(item) do { } while (0)
  31. #define count_compact_events(item, delta) do { } while (0)
  32. #endif
  33. #if defined CONFIG_COMPACTION || defined CONFIG_CMA
  34. #define CREATE_TRACE_POINTS
  35. #include <trace/events/compaction.h>
  36. static unsigned long release_freepages(struct list_head *freelist)
  37. {
  38. struct page *page, *next;
  39. unsigned long count = 0;
  40. list_for_each_entry_safe(page, next, freelist, lru) {
  41. list_del(&page->lru);
  42. __free_page(page);
  43. count++;
  44. }
  45. return count;
  46. }
  47. static void map_pages(struct list_head *list)
  48. {
  49. struct page *page;
  50. list_for_each_entry(page, list, lru) {
  51. arch_alloc_page(page, 0);
  52. kernel_map_pages(page, 1, 1);
  53. }
  54. }
  55. static inline bool migrate_async_suitable(int migratetype)
  56. {
  57. return is_migrate_cma(migratetype) || migratetype == MIGRATE_MOVABLE;
  58. }
  59. #ifdef CONFIG_COMPACTION
  60. /* Returns true if the pageblock should be scanned for pages to isolate. */
  61. static inline bool isolation_suitable(struct compact_control *cc,
  62. struct page *page)
  63. {
  64. if (cc->ignore_skip_hint)
  65. return true;
  66. return !get_pageblock_skip(page);
  67. }
  68. /*
  69. * This function is called to clear all cached information on pageblocks that
  70. * should be skipped for page isolation when the migrate and free page scanner
  71. * meet.
  72. */
  73. static void __reset_isolation_suitable(struct zone *zone)
  74. {
  75. unsigned long start_pfn = zone->zone_start_pfn;
  76. unsigned long end_pfn = zone_end_pfn(zone);
  77. unsigned long pfn;
  78. zone->compact_cached_migrate_pfn = start_pfn;
  79. zone->compact_cached_free_pfn = end_pfn;
  80. zone->compact_blockskip_flush = false;
  81. /* Walk the zone and mark every pageblock as suitable for isolation */
  82. for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
  83. struct page *page;
  84. cond_resched();
  85. if (!pfn_valid(pfn))
  86. continue;
  87. page = pfn_to_page(pfn);
  88. if (zone != page_zone(page))
  89. continue;
  90. clear_pageblock_skip(page);
  91. }
  92. }
  93. void reset_isolation_suitable(pg_data_t *pgdat)
  94. {
  95. int zoneid;
  96. for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
  97. struct zone *zone = &pgdat->node_zones[zoneid];
  98. if (!populated_zone(zone))
  99. continue;
  100. /* Only flush if a full compaction finished recently */
  101. if (zone->compact_blockskip_flush)
  102. __reset_isolation_suitable(zone);
  103. }
  104. }
  105. /*
  106. * If no pages were isolated then mark this pageblock to be skipped in the
  107. * future. The information is later cleared by __reset_isolation_suitable().
  108. */
  109. static void update_pageblock_skip(struct compact_control *cc,
  110. struct page *page, unsigned long nr_isolated,
  111. bool migrate_scanner)
  112. {
  113. struct zone *zone = cc->zone;
  114. if (!page)
  115. return;
  116. if (!nr_isolated) {
  117. unsigned long pfn = page_to_pfn(page);
  118. set_pageblock_skip(page);
  119. /* Update where compaction should restart */
  120. if (migrate_scanner) {
  121. if (!cc->finished_update_migrate &&
  122. pfn > zone->compact_cached_migrate_pfn)
  123. zone->compact_cached_migrate_pfn = pfn;
  124. } else {
  125. if (!cc->finished_update_free &&
  126. pfn < zone->compact_cached_free_pfn)
  127. zone->compact_cached_free_pfn = pfn;
  128. }
  129. }
  130. }
  131. #else
  132. static inline bool isolation_suitable(struct compact_control *cc,
  133. struct page *page)
  134. {
  135. return true;
  136. }
  137. static void update_pageblock_skip(struct compact_control *cc,
  138. struct page *page, unsigned long nr_isolated,
  139. bool migrate_scanner)
  140. {
  141. }
  142. #endif /* CONFIG_COMPACTION */
  143. static inline bool should_release_lock(spinlock_t *lock)
  144. {
  145. return need_resched() || spin_is_contended(lock);
  146. }
  147. /*
  148. * Compaction requires the taking of some coarse locks that are potentially
  149. * very heavily contended. Check if the process needs to be scheduled or
  150. * if the lock is contended. For async compaction, back out in the event
  151. * if contention is severe. For sync compaction, schedule.
  152. *
  153. * Returns true if the lock is held.
  154. * Returns false if the lock is released and compaction should abort
  155. */
  156. static bool compact_checklock_irqsave(spinlock_t *lock, unsigned long *flags,
  157. bool locked, struct compact_control *cc)
  158. {
  159. if (should_release_lock(lock)) {
  160. if (locked) {
  161. spin_unlock_irqrestore(lock, *flags);
  162. locked = false;
  163. }
  164. /* async aborts if taking too long or contended */
  165. if (!cc->sync) {
  166. cc->contended = true;
  167. return false;
  168. }
  169. cond_resched();
  170. }
  171. if (!locked)
  172. spin_lock_irqsave(lock, *flags);
  173. return true;
  174. }
  175. static inline bool compact_trylock_irqsave(spinlock_t *lock,
  176. unsigned long *flags, struct compact_control *cc)
  177. {
  178. return compact_checklock_irqsave(lock, flags, false, cc);
  179. }
  180. /* Returns true if the page is within a block suitable for migration to */
  181. static bool suitable_migration_target(struct page *page)
  182. {
  183. int migratetype = get_pageblock_migratetype(page);
  184. /* Don't interfere with memory hot-remove or the min_free_kbytes blocks */
  185. if (migratetype == MIGRATE_RESERVE)
  186. return false;
  187. if (is_migrate_isolate(migratetype))
  188. return false;
  189. /* If the page is a large free page, then allow migration */
  190. if (PageBuddy(page) && page_order(page) >= pageblock_order)
  191. return true;
  192. /* If the block is MIGRATE_MOVABLE or MIGRATE_CMA, allow migration */
  193. if (migrate_async_suitable(migratetype))
  194. return true;
  195. /* Otherwise skip the block */
  196. return false;
  197. }
  198. /*
  199. * Isolate free pages onto a private freelist. If @strict is true, will abort
  200. * returning 0 on any invalid PFNs or non-free pages inside of the pageblock
  201. * (even though it may still end up isolating some pages).
  202. */
  203. static unsigned long isolate_freepages_block(struct compact_control *cc,
  204. unsigned long blockpfn,
  205. unsigned long end_pfn,
  206. struct list_head *freelist,
  207. bool strict)
  208. {
  209. int nr_scanned = 0, total_isolated = 0;
  210. struct page *cursor, *valid_page = NULL;
  211. unsigned long nr_strict_required = end_pfn - blockpfn;
  212. unsigned long flags;
  213. bool locked = false;
  214. cursor = pfn_to_page(blockpfn);
  215. /* Isolate free pages. */
  216. for (; blockpfn < end_pfn; blockpfn++, cursor++) {
  217. int isolated, i;
  218. struct page *page = cursor;
  219. nr_scanned++;
  220. if (!pfn_valid_within(blockpfn))
  221. continue;
  222. if (!valid_page)
  223. valid_page = page;
  224. if (!PageBuddy(page))
  225. continue;
  226. /*
  227. * The zone lock must be held to isolate freepages.
  228. * Unfortunately this is a very coarse lock and can be
  229. * heavily contended if there are parallel allocations
  230. * or parallel compactions. For async compaction do not
  231. * spin on the lock and we acquire the lock as late as
  232. * possible.
  233. */
  234. locked = compact_checklock_irqsave(&cc->zone->lock, &flags,
  235. locked, cc);
  236. if (!locked)
  237. break;
  238. /* Recheck this is a suitable migration target under lock */
  239. if (!strict && !suitable_migration_target(page))
  240. break;
  241. /* Recheck this is a buddy page under lock */
  242. if (!PageBuddy(page))
  243. continue;
  244. /* Found a free page, break it into order-0 pages */
  245. isolated = split_free_page(page);
  246. if (!isolated && strict)
  247. break;
  248. total_isolated += isolated;
  249. for (i = 0; i < isolated; i++) {
  250. list_add(&page->lru, freelist);
  251. page++;
  252. }
  253. /* If a page was split, advance to the end of it */
  254. if (isolated) {
  255. blockpfn += isolated - 1;
  256. cursor += isolated - 1;
  257. }
  258. }
  259. trace_mm_compaction_isolate_freepages(nr_scanned, total_isolated);
  260. /*
  261. * If strict isolation is requested by CMA then check that all the
  262. * pages requested were isolated. If there were any failures, 0 is
  263. * returned and CMA will fail.
  264. */
  265. if (strict && nr_strict_required > total_isolated)
  266. total_isolated = 0;
  267. if (locked)
  268. spin_unlock_irqrestore(&cc->zone->lock, flags);
  269. /* Update the pageblock-skip if the whole pageblock was scanned */
  270. if (blockpfn == end_pfn)
  271. update_pageblock_skip(cc, valid_page, total_isolated, false);
  272. count_compact_events(COMPACTFREE_SCANNED, nr_scanned);
  273. if (total_isolated)
  274. count_compact_events(COMPACTISOLATED, total_isolated);
  275. return total_isolated;
  276. }
  277. /**
  278. * isolate_freepages_range() - isolate free pages.
  279. * @start_pfn: The first PFN to start isolating.
  280. * @end_pfn: The one-past-last PFN.
  281. *
  282. * Non-free pages, invalid PFNs, or zone boundaries within the
  283. * [start_pfn, end_pfn) range are considered errors, cause function to
  284. * undo its actions and return zero.
  285. *
  286. * Otherwise, function returns one-past-the-last PFN of isolated page
  287. * (which may be greater then end_pfn if end fell in a middle of
  288. * a free page).
  289. */
  290. unsigned long
  291. isolate_freepages_range(struct compact_control *cc,
  292. unsigned long start_pfn, unsigned long end_pfn)
  293. {
  294. unsigned long isolated, pfn, block_end_pfn;
  295. LIST_HEAD(freelist);
  296. for (pfn = start_pfn; pfn < end_pfn; pfn += isolated) {
  297. if (!pfn_valid(pfn) || cc->zone != page_zone(pfn_to_page(pfn)))
  298. break;
  299. /*
  300. * On subsequent iterations ALIGN() is actually not needed,
  301. * but we keep it that we not to complicate the code.
  302. */
  303. block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages);
  304. block_end_pfn = min(block_end_pfn, end_pfn);
  305. isolated = isolate_freepages_block(cc, pfn, block_end_pfn,
  306. &freelist, true);
  307. /*
  308. * In strict mode, isolate_freepages_block() returns 0 if
  309. * there are any holes in the block (ie. invalid PFNs or
  310. * non-free pages).
  311. */
  312. if (!isolated)
  313. break;
  314. /*
  315. * If we managed to isolate pages, it is always (1 << n) *
  316. * pageblock_nr_pages for some non-negative n. (Max order
  317. * page may span two pageblocks).
  318. */
  319. }
  320. /* split_free_page does not map the pages */
  321. map_pages(&freelist);
  322. if (pfn < end_pfn) {
  323. /* Loop terminated early, cleanup. */
  324. release_freepages(&freelist);
  325. return 0;
  326. }
  327. /* We don't use freelists for anything. */
  328. return pfn;
  329. }
  330. /* Update the number of anon and file isolated pages in the zone */
  331. static void acct_isolated(struct zone *zone, bool locked, struct compact_control *cc)
  332. {
  333. struct page *page;
  334. unsigned int count[2] = { 0, };
  335. list_for_each_entry(page, &cc->migratepages, lru)
  336. count[!!page_is_file_cache(page)]++;
  337. /* If locked we can use the interrupt unsafe versions */
  338. if (locked) {
  339. __mod_zone_page_state(zone, NR_ISOLATED_ANON, count[0]);
  340. __mod_zone_page_state(zone, NR_ISOLATED_FILE, count[1]);
  341. } else {
  342. mod_zone_page_state(zone, NR_ISOLATED_ANON, count[0]);
  343. mod_zone_page_state(zone, NR_ISOLATED_FILE, count[1]);
  344. }
  345. }
  346. /* Similar to reclaim, but different enough that they don't share logic */
  347. static bool too_many_isolated(struct zone *zone)
  348. {
  349. unsigned long active, inactive, isolated;
  350. inactive = zone_page_state(zone, NR_INACTIVE_FILE) +
  351. zone_page_state(zone, NR_INACTIVE_ANON);
  352. active = zone_page_state(zone, NR_ACTIVE_FILE) +
  353. zone_page_state(zone, NR_ACTIVE_ANON);
  354. isolated = zone_page_state(zone, NR_ISOLATED_FILE) +
  355. zone_page_state(zone, NR_ISOLATED_ANON);
  356. return isolated > (inactive + active) / 2;
  357. }
  358. /**
  359. * isolate_migratepages_range() - isolate all migrate-able pages in range.
  360. * @zone: Zone pages are in.
  361. * @cc: Compaction control structure.
  362. * @low_pfn: The first PFN of the range.
  363. * @end_pfn: The one-past-the-last PFN of the range.
  364. * @unevictable: true if it allows to isolate unevictable pages
  365. *
  366. * Isolate all pages that can be migrated from the range specified by
  367. * [low_pfn, end_pfn). Returns zero if there is a fatal signal
  368. * pending), otherwise PFN of the first page that was not scanned
  369. * (which may be both less, equal to or more then end_pfn).
  370. *
  371. * Assumes that cc->migratepages is empty and cc->nr_migratepages is
  372. * zero.
  373. *
  374. * Apart from cc->migratepages and cc->nr_migratetypes this function
  375. * does not modify any cc's fields, in particular it does not modify
  376. * (or read for that matter) cc->migrate_pfn.
  377. */
  378. unsigned long
  379. isolate_migratepages_range(struct zone *zone, struct compact_control *cc,
  380. unsigned long low_pfn, unsigned long end_pfn, bool unevictable)
  381. {
  382. unsigned long last_pageblock_nr = 0, pageblock_nr;
  383. unsigned long nr_scanned = 0, nr_isolated = 0;
  384. struct list_head *migratelist = &cc->migratepages;
  385. isolate_mode_t mode = 0;
  386. struct lruvec *lruvec;
  387. unsigned long flags;
  388. bool locked = false;
  389. struct page *page = NULL, *valid_page = NULL;
  390. /*
  391. * Ensure that there are not too many pages isolated from the LRU
  392. * list by either parallel reclaimers or compaction. If there are,
  393. * delay for some time until fewer pages are isolated
  394. */
  395. while (unlikely(too_many_isolated(zone))) {
  396. /* async migration should just abort */
  397. if (!cc->sync)
  398. return 0;
  399. congestion_wait(BLK_RW_ASYNC, HZ/10);
  400. if (fatal_signal_pending(current))
  401. return 0;
  402. }
  403. /* Time to isolate some pages for migration */
  404. cond_resched();
  405. for (; low_pfn < end_pfn; low_pfn++) {
  406. /* give a chance to irqs before checking need_resched() */
  407. if (locked && !((low_pfn+1) % SWAP_CLUSTER_MAX)) {
  408. if (should_release_lock(&zone->lru_lock)) {
  409. spin_unlock_irqrestore(&zone->lru_lock, flags);
  410. locked = false;
  411. }
  412. }
  413. /*
  414. * migrate_pfn does not necessarily start aligned to a
  415. * pageblock. Ensure that pfn_valid is called when moving
  416. * into a new MAX_ORDER_NR_PAGES range in case of large
  417. * memory holes within the zone
  418. */
  419. if ((low_pfn & (MAX_ORDER_NR_PAGES - 1)) == 0) {
  420. if (!pfn_valid(low_pfn)) {
  421. low_pfn += MAX_ORDER_NR_PAGES - 1;
  422. continue;
  423. }
  424. }
  425. if (!pfn_valid_within(low_pfn))
  426. continue;
  427. nr_scanned++;
  428. /*
  429. * Get the page and ensure the page is within the same zone.
  430. * See the comment in isolate_freepages about overlapping
  431. * nodes. It is deliberate that the new zone lock is not taken
  432. * as memory compaction should not move pages between nodes.
  433. */
  434. page = pfn_to_page(low_pfn);
  435. if (page_zone(page) != zone)
  436. continue;
  437. if (!valid_page)
  438. valid_page = page;
  439. /* If isolation recently failed, do not retry */
  440. pageblock_nr = low_pfn >> pageblock_order;
  441. if (!isolation_suitable(cc, page))
  442. goto next_pageblock;
  443. /* Skip if free */
  444. if (PageBuddy(page))
  445. continue;
  446. /*
  447. * For async migration, also only scan in MOVABLE blocks. Async
  448. * migration is optimistic to see if the minimum amount of work
  449. * satisfies the allocation
  450. */
  451. if (!cc->sync && last_pageblock_nr != pageblock_nr &&
  452. !migrate_async_suitable(get_pageblock_migratetype(page))) {
  453. cc->finished_update_migrate = true;
  454. goto next_pageblock;
  455. }
  456. /*
  457. * Check may be lockless but that's ok as we recheck later.
  458. * It's possible to migrate LRU pages and balloon pages
  459. * Skip any other type of page
  460. */
  461. if (!PageLRU(page)) {
  462. if (unlikely(balloon_page_movable(page))) {
  463. if (locked && balloon_page_isolate(page)) {
  464. /* Successfully isolated */
  465. cc->finished_update_migrate = true;
  466. list_add(&page->lru, migratelist);
  467. cc->nr_migratepages++;
  468. nr_isolated++;
  469. goto check_compact_cluster;
  470. }
  471. }
  472. continue;
  473. }
  474. /*
  475. * PageLRU is set. lru_lock normally excludes isolation
  476. * splitting and collapsing (collapsing has already happened
  477. * if PageLRU is set) but the lock is not necessarily taken
  478. * here and it is wasteful to take it just to check transhuge.
  479. * Check TransHuge without lock and skip the whole pageblock if
  480. * it's either a transhuge or hugetlbfs page, as calling
  481. * compound_order() without preventing THP from splitting the
  482. * page underneath us may return surprising results.
  483. */
  484. if (PageTransHuge(page)) {
  485. if (!locked)
  486. goto next_pageblock;
  487. low_pfn += (1 << compound_order(page)) - 1;
  488. continue;
  489. }
  490. /* Check if it is ok to still hold the lock */
  491. locked = compact_checklock_irqsave(&zone->lru_lock, &flags,
  492. locked, cc);
  493. if (!locked || fatal_signal_pending(current))
  494. break;
  495. /* Recheck PageLRU and PageTransHuge under lock */
  496. if (!PageLRU(page))
  497. continue;
  498. if (PageTransHuge(page)) {
  499. low_pfn += (1 << compound_order(page)) - 1;
  500. continue;
  501. }
  502. if (!cc->sync)
  503. mode |= ISOLATE_ASYNC_MIGRATE;
  504. if (unevictable)
  505. mode |= ISOLATE_UNEVICTABLE;
  506. lruvec = mem_cgroup_page_lruvec(page, zone);
  507. /* Try isolate the page */
  508. if (__isolate_lru_page(page, mode) != 0)
  509. continue;
  510. VM_BUG_ON(PageTransCompound(page));
  511. /* Successfully isolated */
  512. cc->finished_update_migrate = true;
  513. del_page_from_lru_list(page, lruvec, page_lru(page));
  514. list_add(&page->lru, migratelist);
  515. cc->nr_migratepages++;
  516. nr_isolated++;
  517. check_compact_cluster:
  518. /* Avoid isolating too much */
  519. if (cc->nr_migratepages == COMPACT_CLUSTER_MAX) {
  520. ++low_pfn;
  521. break;
  522. }
  523. continue;
  524. next_pageblock:
  525. low_pfn = ALIGN(low_pfn + 1, pageblock_nr_pages) - 1;
  526. last_pageblock_nr = pageblock_nr;
  527. }
  528. acct_isolated(zone, locked, cc);
  529. if (locked)
  530. spin_unlock_irqrestore(&zone->lru_lock, flags);
  531. /* Update the pageblock-skip if the whole pageblock was scanned */
  532. if (low_pfn == end_pfn)
  533. update_pageblock_skip(cc, valid_page, nr_isolated, true);
  534. trace_mm_compaction_isolate_migratepages(nr_scanned, nr_isolated);
  535. count_compact_events(COMPACTMIGRATE_SCANNED, nr_scanned);
  536. if (nr_isolated)
  537. count_compact_events(COMPACTISOLATED, nr_isolated);
  538. return low_pfn;
  539. }
  540. #endif /* CONFIG_COMPACTION || CONFIG_CMA */
  541. #ifdef CONFIG_COMPACTION
  542. /*
  543. * Based on information in the current compact_control, find blocks
  544. * suitable for isolating free pages from and then isolate them.
  545. */
  546. static void isolate_freepages(struct zone *zone,
  547. struct compact_control *cc)
  548. {
  549. struct page *page;
  550. unsigned long high_pfn, low_pfn, pfn, z_end_pfn, end_pfn;
  551. int nr_freepages = cc->nr_freepages;
  552. struct list_head *freelist = &cc->freepages;
  553. /*
  554. * Initialise the free scanner. The starting point is where we last
  555. * scanned from (or the end of the zone if starting). The low point
  556. * is the end of the pageblock the migration scanner is using.
  557. */
  558. pfn = cc->free_pfn;
  559. low_pfn = cc->migrate_pfn + pageblock_nr_pages;
  560. /*
  561. * Take care that if the migration scanner is at the end of the zone
  562. * that the free scanner does not accidentally move to the next zone
  563. * in the next isolation cycle.
  564. */
  565. high_pfn = min(low_pfn, pfn);
  566. z_end_pfn = zone_end_pfn(zone);
  567. /*
  568. * Isolate free pages until enough are available to migrate the
  569. * pages on cc->migratepages. We stop searching if the migrate
  570. * and free page scanners meet or enough free pages are isolated.
  571. */
  572. for (; pfn > low_pfn && cc->nr_migratepages > nr_freepages;
  573. pfn -= pageblock_nr_pages) {
  574. unsigned long isolated;
  575. /*
  576. * This can iterate a massively long zone without finding any
  577. * suitable migration targets, so periodically check if we need
  578. * to schedule.
  579. */
  580. cond_resched();
  581. if (!pfn_valid(pfn))
  582. continue;
  583. /*
  584. * Check for overlapping nodes/zones. It's possible on some
  585. * configurations to have a setup like
  586. * node0 node1 node0
  587. * i.e. it's possible that all pages within a zones range of
  588. * pages do not belong to a single zone.
  589. */
  590. page = pfn_to_page(pfn);
  591. if (page_zone(page) != zone)
  592. continue;
  593. /* Check the block is suitable for migration */
  594. if (!suitable_migration_target(page))
  595. continue;
  596. /* If isolation recently failed, do not retry */
  597. if (!isolation_suitable(cc, page))
  598. continue;
  599. /* Found a block suitable for isolating free pages from */
  600. isolated = 0;
  601. /*
  602. * As pfn may not start aligned, pfn+pageblock_nr_page
  603. * may cross a MAX_ORDER_NR_PAGES boundary and miss
  604. * a pfn_valid check. Ensure isolate_freepages_block()
  605. * only scans within a pageblock
  606. */
  607. end_pfn = ALIGN(pfn + 1, pageblock_nr_pages);
  608. end_pfn = min(end_pfn, z_end_pfn);
  609. isolated = isolate_freepages_block(cc, pfn, end_pfn,
  610. freelist, false);
  611. nr_freepages += isolated;
  612. /*
  613. * Record the highest PFN we isolated pages from. When next
  614. * looking for free pages, the search will restart here as
  615. * page migration may have returned some pages to the allocator
  616. */
  617. if (isolated) {
  618. cc->finished_update_free = true;
  619. high_pfn = max(high_pfn, pfn);
  620. }
  621. }
  622. /* split_free_page does not map the pages */
  623. map_pages(freelist);
  624. cc->free_pfn = high_pfn;
  625. cc->nr_freepages = nr_freepages;
  626. }
  627. /*
  628. * This is a migrate-callback that "allocates" freepages by taking pages
  629. * from the isolated freelists in the block we are migrating to.
  630. */
  631. static struct page *compaction_alloc(struct page *migratepage,
  632. unsigned long data,
  633. int **result)
  634. {
  635. struct compact_control *cc = (struct compact_control *)data;
  636. struct page *freepage;
  637. /* Isolate free pages if necessary */
  638. if (list_empty(&cc->freepages)) {
  639. isolate_freepages(cc->zone, cc);
  640. if (list_empty(&cc->freepages))
  641. return NULL;
  642. }
  643. freepage = list_entry(cc->freepages.next, struct page, lru);
  644. list_del(&freepage->lru);
  645. cc->nr_freepages--;
  646. return freepage;
  647. }
  648. /*
  649. * We cannot control nr_migratepages and nr_freepages fully when migration is
  650. * running as migrate_pages() has no knowledge of compact_control. When
  651. * migration is complete, we count the number of pages on the lists by hand.
  652. */
  653. static void update_nr_listpages(struct compact_control *cc)
  654. {
  655. int nr_migratepages = 0;
  656. int nr_freepages = 0;
  657. struct page *page;
  658. list_for_each_entry(page, &cc->migratepages, lru)
  659. nr_migratepages++;
  660. list_for_each_entry(page, &cc->freepages, lru)
  661. nr_freepages++;
  662. cc->nr_migratepages = nr_migratepages;
  663. cc->nr_freepages = nr_freepages;
  664. }
  665. /* possible outcome of isolate_migratepages */
  666. typedef enum {
  667. ISOLATE_ABORT, /* Abort compaction now */
  668. ISOLATE_NONE, /* No pages isolated, continue scanning */
  669. ISOLATE_SUCCESS, /* Pages isolated, migrate */
  670. } isolate_migrate_t;
  671. /*
  672. * Isolate all pages that can be migrated from the block pointed to by
  673. * the migrate scanner within compact_control.
  674. */
  675. static isolate_migrate_t isolate_migratepages(struct zone *zone,
  676. struct compact_control *cc)
  677. {
  678. unsigned long low_pfn, end_pfn;
  679. /* Do not scan outside zone boundaries */
  680. low_pfn = max(cc->migrate_pfn, zone->zone_start_pfn);
  681. /* Only scan within a pageblock boundary */
  682. end_pfn = ALIGN(low_pfn + 1, pageblock_nr_pages);
  683. /* Do not cross the free scanner or scan within a memory hole */
  684. if (end_pfn > cc->free_pfn || !pfn_valid(low_pfn)) {
  685. cc->migrate_pfn = end_pfn;
  686. return ISOLATE_NONE;
  687. }
  688. /* Perform the isolation */
  689. low_pfn = isolate_migratepages_range(zone, cc, low_pfn, end_pfn, false);
  690. if (!low_pfn || cc->contended)
  691. return ISOLATE_ABORT;
  692. cc->migrate_pfn = low_pfn;
  693. return ISOLATE_SUCCESS;
  694. }
  695. static int compact_finished(struct zone *zone,
  696. struct compact_control *cc)
  697. {
  698. unsigned int order;
  699. unsigned long watermark;
  700. if (fatal_signal_pending(current))
  701. return COMPACT_PARTIAL;
  702. /* Compaction run completes if the migrate and free scanner meet */
  703. if (cc->free_pfn <= cc->migrate_pfn) {
  704. /*
  705. * Mark that the PG_migrate_skip information should be cleared
  706. * by kswapd when it goes to sleep. kswapd does not set the
  707. * flag itself as the decision to be clear should be directly
  708. * based on an allocation request.
  709. */
  710. if (!current_is_kswapd())
  711. zone->compact_blockskip_flush = true;
  712. return COMPACT_COMPLETE;
  713. }
  714. /*
  715. * order == -1 is expected when compacting via
  716. * /proc/sys/vm/compact_memory
  717. */
  718. if (cc->order == -1)
  719. return COMPACT_CONTINUE;
  720. /* Compaction run is not finished if the watermark is not met */
  721. watermark = low_wmark_pages(zone);
  722. watermark += (1 << cc->order);
  723. if (!zone_watermark_ok(zone, cc->order, watermark, 0, 0))
  724. return COMPACT_CONTINUE;
  725. /* Direct compactor: Is a suitable page free? */
  726. for (order = cc->order; order < MAX_ORDER; order++) {
  727. struct free_area *area = &zone->free_area[order];
  728. /* Job done if page is free of the right migratetype */
  729. if (!list_empty(&area->free_list[cc->migratetype]))
  730. return COMPACT_PARTIAL;
  731. /* Job done if allocation would set block type */
  732. if (cc->order >= pageblock_order && area->nr_free)
  733. return COMPACT_PARTIAL;
  734. }
  735. return COMPACT_CONTINUE;
  736. }
  737. /*
  738. * compaction_suitable: Is this suitable to run compaction on this zone now?
  739. * Returns
  740. * COMPACT_SKIPPED - If there are too few free pages for compaction
  741. * COMPACT_PARTIAL - If the allocation would succeed without compaction
  742. * COMPACT_CONTINUE - If compaction should run now
  743. */
  744. unsigned long compaction_suitable(struct zone *zone, int order)
  745. {
  746. int fragindex;
  747. unsigned long watermark;
  748. /*
  749. * order == -1 is expected when compacting via
  750. * /proc/sys/vm/compact_memory
  751. */
  752. if (order == -1)
  753. return COMPACT_CONTINUE;
  754. /*
  755. * Watermarks for order-0 must be met for compaction. Note the 2UL.
  756. * This is because during migration, copies of pages need to be
  757. * allocated and for a short time, the footprint is higher
  758. */
  759. watermark = low_wmark_pages(zone) + (2UL << order);
  760. if (!zone_watermark_ok(zone, 0, watermark, 0, 0))
  761. return COMPACT_SKIPPED;
  762. /*
  763. * fragmentation index determines if allocation failures are due to
  764. * low memory or external fragmentation
  765. *
  766. * index of -1000 implies allocations might succeed depending on
  767. * watermarks
  768. * index towards 0 implies failure is due to lack of memory
  769. * index towards 1000 implies failure is due to fragmentation
  770. *
  771. * Only compact if a failure would be due to fragmentation.
  772. */
  773. fragindex = fragmentation_index(zone, order);
  774. if (fragindex >= 0 && fragindex <= sysctl_extfrag_threshold)
  775. return COMPACT_SKIPPED;
  776. if (fragindex == -1000 && zone_watermark_ok(zone, order, watermark,
  777. 0, 0))
  778. return COMPACT_PARTIAL;
  779. return COMPACT_CONTINUE;
  780. }
  781. static int compact_zone(struct zone *zone, struct compact_control *cc)
  782. {
  783. int ret;
  784. unsigned long start_pfn = zone->zone_start_pfn;
  785. unsigned long end_pfn = zone_end_pfn(zone);
  786. ret = compaction_suitable(zone, cc->order);
  787. switch (ret) {
  788. case COMPACT_PARTIAL:
  789. case COMPACT_SKIPPED:
  790. /* Compaction is likely to fail */
  791. return ret;
  792. case COMPACT_CONTINUE:
  793. /* Fall through to compaction */
  794. ;
  795. }
  796. /*
  797. * Setup to move all movable pages to the end of the zone. Used cached
  798. * information on where the scanners should start but check that it
  799. * is initialised by ensuring the values are within zone boundaries.
  800. */
  801. cc->migrate_pfn = zone->compact_cached_migrate_pfn;
  802. cc->free_pfn = zone->compact_cached_free_pfn;
  803. if (cc->free_pfn < start_pfn || cc->free_pfn > end_pfn) {
  804. cc->free_pfn = end_pfn & ~(pageblock_nr_pages-1);
  805. zone->compact_cached_free_pfn = cc->free_pfn;
  806. }
  807. if (cc->migrate_pfn < start_pfn || cc->migrate_pfn > end_pfn) {
  808. cc->migrate_pfn = start_pfn;
  809. zone->compact_cached_migrate_pfn = cc->migrate_pfn;
  810. }
  811. /*
  812. * Clear pageblock skip if there were failures recently and compaction
  813. * is about to be retried after being deferred. kswapd does not do
  814. * this reset as it'll reset the cached information when going to sleep.
  815. */
  816. if (compaction_restarting(zone, cc->order) && !current_is_kswapd())
  817. __reset_isolation_suitable(zone);
  818. migrate_prep_local();
  819. while ((ret = compact_finished(zone, cc)) == COMPACT_CONTINUE) {
  820. unsigned long nr_migrate, nr_remaining;
  821. int err;
  822. switch (isolate_migratepages(zone, cc)) {
  823. case ISOLATE_ABORT:
  824. ret = COMPACT_PARTIAL;
  825. putback_movable_pages(&cc->migratepages);
  826. cc->nr_migratepages = 0;
  827. goto out;
  828. case ISOLATE_NONE:
  829. continue;
  830. case ISOLATE_SUCCESS:
  831. ;
  832. }
  833. nr_migrate = cc->nr_migratepages;
  834. err = migrate_pages(&cc->migratepages, compaction_alloc,
  835. (unsigned long)cc,
  836. cc->sync ? MIGRATE_SYNC_LIGHT : MIGRATE_ASYNC,
  837. MR_COMPACTION);
  838. update_nr_listpages(cc);
  839. nr_remaining = cc->nr_migratepages;
  840. trace_mm_compaction_migratepages(nr_migrate - nr_remaining,
  841. nr_remaining);
  842. /* Release isolated pages not migrated */
  843. if (err) {
  844. putback_movable_pages(&cc->migratepages);
  845. cc->nr_migratepages = 0;
  846. if (err == -ENOMEM) {
  847. ret = COMPACT_PARTIAL;
  848. goto out;
  849. }
  850. }
  851. }
  852. out:
  853. /* Release free pages and check accounting */
  854. cc->nr_freepages -= release_freepages(&cc->freepages);
  855. VM_BUG_ON(cc->nr_freepages != 0);
  856. return ret;
  857. }
  858. static unsigned long compact_zone_order(struct zone *zone,
  859. int order, gfp_t gfp_mask,
  860. bool sync, bool *contended)
  861. {
  862. unsigned long ret;
  863. struct compact_control cc = {
  864. .nr_freepages = 0,
  865. .nr_migratepages = 0,
  866. .order = order,
  867. .migratetype = allocflags_to_migratetype(gfp_mask),
  868. .zone = zone,
  869. .sync = sync,
  870. };
  871. INIT_LIST_HEAD(&cc.freepages);
  872. INIT_LIST_HEAD(&cc.migratepages);
  873. ret = compact_zone(zone, &cc);
  874. VM_BUG_ON(!list_empty(&cc.freepages));
  875. VM_BUG_ON(!list_empty(&cc.migratepages));
  876. *contended = cc.contended;
  877. return ret;
  878. }
  879. int sysctl_extfrag_threshold = 500;
  880. /**
  881. * try_to_compact_pages - Direct compact to satisfy a high-order allocation
  882. * @zonelist: The zonelist used for the current allocation
  883. * @order: The order of the current allocation
  884. * @gfp_mask: The GFP mask of the current allocation
  885. * @nodemask: The allowed nodes to allocate from
  886. * @sync: Whether migration is synchronous or not
  887. * @contended: Return value that is true if compaction was aborted due to lock contention
  888. * @page: Optionally capture a free page of the requested order during compaction
  889. *
  890. * This is the main entry point for direct page compaction.
  891. */
  892. unsigned long try_to_compact_pages(struct zonelist *zonelist,
  893. int order, gfp_t gfp_mask, nodemask_t *nodemask,
  894. bool sync, bool *contended)
  895. {
  896. enum zone_type high_zoneidx = gfp_zone(gfp_mask);
  897. int may_enter_fs = gfp_mask & __GFP_FS;
  898. int may_perform_io = gfp_mask & __GFP_IO;
  899. struct zoneref *z;
  900. struct zone *zone;
  901. int rc = COMPACT_SKIPPED;
  902. int alloc_flags = 0;
  903. /* Check if the GFP flags allow compaction */
  904. if (!order || !may_enter_fs || !may_perform_io)
  905. return rc;
  906. count_compact_event(COMPACTSTALL);
  907. #ifdef CONFIG_CMA
  908. if (allocflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
  909. alloc_flags |= ALLOC_CMA;
  910. #endif
  911. /* Compact each zone in the list */
  912. for_each_zone_zonelist_nodemask(zone, z, zonelist, high_zoneidx,
  913. nodemask) {
  914. int status;
  915. status = compact_zone_order(zone, order, gfp_mask, sync,
  916. contended);
  917. rc = max(status, rc);
  918. /* If a normal allocation would succeed, stop compacting */
  919. if (zone_watermark_ok(zone, order, low_wmark_pages(zone), 0,
  920. alloc_flags))
  921. break;
  922. }
  923. return rc;
  924. }
  925. /* Compact all zones within a node */
  926. static void __compact_pgdat(pg_data_t *pgdat, struct compact_control *cc)
  927. {
  928. int zoneid;
  929. struct zone *zone;
  930. for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
  931. zone = &pgdat->node_zones[zoneid];
  932. if (!populated_zone(zone))
  933. continue;
  934. cc->nr_freepages = 0;
  935. cc->nr_migratepages = 0;
  936. cc->zone = zone;
  937. INIT_LIST_HEAD(&cc->freepages);
  938. INIT_LIST_HEAD(&cc->migratepages);
  939. if (cc->order == -1 || !compaction_deferred(zone, cc->order))
  940. compact_zone(zone, cc);
  941. if (cc->order > 0) {
  942. int ok = zone_watermark_ok(zone, cc->order,
  943. low_wmark_pages(zone), 0, 0);
  944. if (ok && cc->order >= zone->compact_order_failed)
  945. zone->compact_order_failed = cc->order + 1;
  946. /* Currently async compaction is never deferred. */
  947. else if (!ok && cc->sync)
  948. defer_compaction(zone, cc->order);
  949. }
  950. VM_BUG_ON(!list_empty(&cc->freepages));
  951. VM_BUG_ON(!list_empty(&cc->migratepages));
  952. }
  953. }
  954. void compact_pgdat(pg_data_t *pgdat, int order)
  955. {
  956. struct compact_control cc = {
  957. .order = order,
  958. .sync = false,
  959. };
  960. if (!order)
  961. return;
  962. __compact_pgdat(pgdat, &cc);
  963. }
  964. static void compact_node(int nid)
  965. {
  966. struct compact_control cc = {
  967. .order = -1,
  968. .sync = true,
  969. };
  970. __compact_pgdat(NODE_DATA(nid), &cc);
  971. }
  972. /* Compact all nodes in the system */
  973. static void compact_nodes(void)
  974. {
  975. int nid;
  976. /* Flush pending updates to the LRU lists */
  977. lru_add_drain_all();
  978. for_each_online_node(nid)
  979. compact_node(nid);
  980. }
  981. /* The written value is actually unused, all memory is compacted */
  982. int sysctl_compact_memory;
  983. /* This is the entry point for compacting all nodes via /proc/sys/vm */
  984. int sysctl_compaction_handler(struct ctl_table *table, int write,
  985. void __user *buffer, size_t *length, loff_t *ppos)
  986. {
  987. if (write)
  988. compact_nodes();
  989. return 0;
  990. }
  991. int sysctl_extfrag_handler(struct ctl_table *table, int write,
  992. void __user *buffer, size_t *length, loff_t *ppos)
  993. {
  994. proc_dointvec_minmax(table, write, buffer, length, ppos);
  995. return 0;
  996. }
  997. #if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)
  998. ssize_t sysfs_compact_node(struct device *dev,
  999. struct device_attribute *attr,
  1000. const char *buf, size_t count)
  1001. {
  1002. int nid = dev->id;
  1003. if (nid >= 0 && nid < nr_node_ids && node_online(nid)) {
  1004. /* Flush pending updates to the LRU lists */
  1005. lru_add_drain_all();
  1006. compact_node(nid);
  1007. }
  1008. return count;
  1009. }
  1010. static DEVICE_ATTR(compact, S_IWUSR, NULL, sysfs_compact_node);
  1011. int compaction_register_node(struct node *node)
  1012. {
  1013. return device_create_file(&node->dev, &dev_attr_compact);
  1014. }
  1015. void compaction_unregister_node(struct node *node)
  1016. {
  1017. return device_remove_file(&node->dev, &dev_attr_compact);
  1018. }
  1019. #endif /* CONFIG_SYSFS && CONFIG_NUMA */
  1020. #endif /* CONFIG_COMPACTION */