migrate.c 46 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833
  1. /*
  2. * Memory Migration functionality - linux/mm/migration.c
  3. *
  4. * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter
  5. *
  6. * Page migration was first developed in the context of the memory hotplug
  7. * project. The main authors of the migration code are:
  8. *
  9. * IWAMOTO Toshihiro <iwamoto@valinux.co.jp>
  10. * Hirokazu Takahashi <taka@valinux.co.jp>
  11. * Dave Hansen <haveblue@us.ibm.com>
  12. * Christoph Lameter
  13. */
  14. #include <linux/migrate.h>
  15. #include <linux/export.h>
  16. #include <linux/swap.h>
  17. #include <linux/swapops.h>
  18. #include <linux/pagemap.h>
  19. #include <linux/buffer_head.h>
  20. #include <linux/mm_inline.h>
  21. #include <linux/nsproxy.h>
  22. #include <linux/pagevec.h>
  23. #include <linux/ksm.h>
  24. #include <linux/rmap.h>
  25. #include <linux/topology.h>
  26. #include <linux/cpu.h>
  27. #include <linux/cpuset.h>
  28. #include <linux/writeback.h>
  29. #include <linux/mempolicy.h>
  30. #include <linux/vmalloc.h>
  31. #include <linux/security.h>
  32. #include <linux/memcontrol.h>
  33. #include <linux/syscalls.h>
  34. #include <linux/hugetlb.h>
  35. #include <linux/hugetlb_cgroup.h>
  36. #include <linux/gfp.h>
  37. #include <linux/balloon_compaction.h>
  38. #include <asm/tlbflush.h>
  39. #define CREATE_TRACE_POINTS
  40. #include <trace/events/migrate.h>
  41. #include "internal.h"
  42. /*
  43. * migrate_prep() needs to be called before we start compiling a list of pages
  44. * to be migrated using isolate_lru_page(). If scheduling work on other CPUs is
  45. * undesirable, use migrate_prep_local()
  46. */
  47. int migrate_prep(void)
  48. {
  49. /*
  50. * Clear the LRU lists so pages can be isolated.
  51. * Note that pages may be moved off the LRU after we have
  52. * drained them. Those pages will fail to migrate like other
  53. * pages that may be busy.
  54. */
  55. lru_add_drain_all();
  56. return 0;
  57. }
  58. /* Do the necessary work of migrate_prep but not if it involves other CPUs */
  59. int migrate_prep_local(void)
  60. {
  61. lru_add_drain();
  62. return 0;
  63. }
  64. /*
  65. * Add isolated pages on the list back to the LRU under page lock
  66. * to avoid leaking evictable pages back onto unevictable list.
  67. */
  68. void putback_lru_pages(struct list_head *l)
  69. {
  70. struct page *page;
  71. struct page *page2;
  72. list_for_each_entry_safe(page, page2, l, lru) {
  73. list_del(&page->lru);
  74. dec_zone_page_state(page, NR_ISOLATED_ANON +
  75. page_is_file_cache(page));
  76. putback_lru_page(page);
  77. }
  78. }
  79. /*
  80. * Put previously isolated pages back onto the appropriate lists
  81. * from where they were once taken off for compaction/migration.
  82. *
  83. * This function shall be used instead of putback_lru_pages(),
  84. * whenever the isolated pageset has been built by isolate_migratepages_range()
  85. */
  86. void putback_movable_pages(struct list_head *l)
  87. {
  88. struct page *page;
  89. struct page *page2;
  90. list_for_each_entry_safe(page, page2, l, lru) {
  91. if (unlikely(PageHuge(page))) {
  92. putback_active_hugepage(page);
  93. continue;
  94. }
  95. list_del(&page->lru);
  96. dec_zone_page_state(page, NR_ISOLATED_ANON +
  97. page_is_file_cache(page));
  98. if (unlikely(isolated_balloon_page(page)))
  99. balloon_page_putback(page);
  100. else
  101. putback_lru_page(page);
  102. }
  103. }
  104. /*
  105. * Restore a potential migration pte to a working pte entry
  106. */
  107. static int remove_migration_pte(struct page *new, struct vm_area_struct *vma,
  108. unsigned long addr, void *old)
  109. {
  110. struct mm_struct *mm = vma->vm_mm;
  111. swp_entry_t entry;
  112. pmd_t *pmd;
  113. pte_t *ptep, pte;
  114. spinlock_t *ptl;
  115. if (unlikely(PageHuge(new))) {
  116. ptep = huge_pte_offset(mm, addr);
  117. if (!ptep)
  118. goto out;
  119. ptl = huge_pte_lockptr(hstate_vma(vma), mm, ptep);
  120. } else {
  121. pmd = mm_find_pmd(mm, addr);
  122. if (!pmd)
  123. goto out;
  124. if (pmd_trans_huge(*pmd))
  125. goto out;
  126. ptep = pte_offset_map(pmd, addr);
  127. /*
  128. * Peek to check is_swap_pte() before taking ptlock? No, we
  129. * can race mremap's move_ptes(), which skips anon_vma lock.
  130. */
  131. ptl = pte_lockptr(mm, pmd);
  132. }
  133. spin_lock(ptl);
  134. pte = *ptep;
  135. if (!is_swap_pte(pte))
  136. goto unlock;
  137. entry = pte_to_swp_entry(pte);
  138. if (!is_migration_entry(entry) ||
  139. migration_entry_to_page(entry) != old)
  140. goto unlock;
  141. get_page(new);
  142. pte = pte_mkold(mk_pte(new, vma->vm_page_prot));
  143. if (pte_swp_soft_dirty(*ptep))
  144. pte = pte_mksoft_dirty(pte);
  145. if (is_write_migration_entry(entry))
  146. pte = pte_mkwrite(pte);
  147. #ifdef CONFIG_HUGETLB_PAGE
  148. if (PageHuge(new)) {
  149. pte = pte_mkhuge(pte);
  150. pte = arch_make_huge_pte(pte, vma, new, 0);
  151. }
  152. #endif
  153. flush_dcache_page(new);
  154. set_pte_at(mm, addr, ptep, pte);
  155. if (PageHuge(new)) {
  156. if (PageAnon(new))
  157. hugepage_add_anon_rmap(new, vma, addr);
  158. else
  159. page_dup_rmap(new);
  160. } else if (PageAnon(new))
  161. page_add_anon_rmap(new, vma, addr);
  162. else
  163. page_add_file_rmap(new);
  164. /* No need to invalidate - it was non-present before */
  165. update_mmu_cache(vma, addr, ptep);
  166. unlock:
  167. pte_unmap_unlock(ptep, ptl);
  168. out:
  169. return SWAP_AGAIN;
  170. }
  171. /*
  172. * Get rid of all migration entries and replace them by
  173. * references to the indicated page.
  174. */
  175. static void remove_migration_ptes(struct page *old, struct page *new)
  176. {
  177. rmap_walk(new, remove_migration_pte, old);
  178. }
  179. /*
  180. * Something used the pte of a page under migration. We need to
  181. * get to the page and wait until migration is finished.
  182. * When we return from this function the fault will be retried.
  183. */
  184. static void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
  185. spinlock_t *ptl)
  186. {
  187. pte_t pte;
  188. swp_entry_t entry;
  189. struct page *page;
  190. spin_lock(ptl);
  191. pte = *ptep;
  192. if (!is_swap_pte(pte))
  193. goto out;
  194. entry = pte_to_swp_entry(pte);
  195. if (!is_migration_entry(entry))
  196. goto out;
  197. page = migration_entry_to_page(entry);
  198. /*
  199. * Once radix-tree replacement of page migration started, page_count
  200. * *must* be zero. And, we don't want to call wait_on_page_locked()
  201. * against a page without get_page().
  202. * So, we use get_page_unless_zero(), here. Even failed, page fault
  203. * will occur again.
  204. */
  205. if (!get_page_unless_zero(page))
  206. goto out;
  207. pte_unmap_unlock(ptep, ptl);
  208. wait_on_page_locked(page);
  209. put_page(page);
  210. return;
  211. out:
  212. pte_unmap_unlock(ptep, ptl);
  213. }
  214. void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
  215. unsigned long address)
  216. {
  217. spinlock_t *ptl = pte_lockptr(mm, pmd);
  218. pte_t *ptep = pte_offset_map(pmd, address);
  219. __migration_entry_wait(mm, ptep, ptl);
  220. }
  221. void migration_entry_wait_huge(struct vm_area_struct *vma,
  222. struct mm_struct *mm, pte_t *pte)
  223. {
  224. spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), mm, pte);
  225. __migration_entry_wait(mm, pte, ptl);
  226. }
  227. #ifdef CONFIG_BLOCK
  228. /* Returns true if all buffers are successfully locked */
  229. static bool buffer_migrate_lock_buffers(struct buffer_head *head,
  230. enum migrate_mode mode)
  231. {
  232. struct buffer_head *bh = head;
  233. /* Simple case, sync compaction */
  234. if (mode != MIGRATE_ASYNC) {
  235. do {
  236. get_bh(bh);
  237. lock_buffer(bh);
  238. bh = bh->b_this_page;
  239. } while (bh != head);
  240. return true;
  241. }
  242. /* async case, we cannot block on lock_buffer so use trylock_buffer */
  243. do {
  244. get_bh(bh);
  245. if (!trylock_buffer(bh)) {
  246. /*
  247. * We failed to lock the buffer and cannot stall in
  248. * async migration. Release the taken locks
  249. */
  250. struct buffer_head *failed_bh = bh;
  251. put_bh(failed_bh);
  252. bh = head;
  253. while (bh != failed_bh) {
  254. unlock_buffer(bh);
  255. put_bh(bh);
  256. bh = bh->b_this_page;
  257. }
  258. return false;
  259. }
  260. bh = bh->b_this_page;
  261. } while (bh != head);
  262. return true;
  263. }
  264. #else
  265. static inline bool buffer_migrate_lock_buffers(struct buffer_head *head,
  266. enum migrate_mode mode)
  267. {
  268. return true;
  269. }
  270. #endif /* CONFIG_BLOCK */
  271. /*
  272. * Replace the page in the mapping.
  273. *
  274. * The number of remaining references must be:
  275. * 1 for anonymous pages without a mapping
  276. * 2 for pages with a mapping
  277. * 3 for pages with a mapping and PagePrivate/PagePrivate2 set.
  278. */
  279. int migrate_page_move_mapping(struct address_space *mapping,
  280. struct page *newpage, struct page *page,
  281. struct buffer_head *head, enum migrate_mode mode)
  282. {
  283. int expected_count = 0;
  284. void **pslot;
  285. if (!mapping) {
  286. /* Anonymous page without mapping */
  287. if (page_count(page) != 1)
  288. return -EAGAIN;
  289. return MIGRATEPAGE_SUCCESS;
  290. }
  291. spin_lock_irq(&mapping->tree_lock);
  292. pslot = radix_tree_lookup_slot(&mapping->page_tree,
  293. page_index(page));
  294. expected_count = 2 + page_has_private(page);
  295. if (page_count(page) != expected_count ||
  296. radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) {
  297. spin_unlock_irq(&mapping->tree_lock);
  298. return -EAGAIN;
  299. }
  300. if (!page_freeze_refs(page, expected_count)) {
  301. spin_unlock_irq(&mapping->tree_lock);
  302. return -EAGAIN;
  303. }
  304. /*
  305. * In the async migration case of moving a page with buffers, lock the
  306. * buffers using trylock before the mapping is moved. If the mapping
  307. * was moved, we later failed to lock the buffers and could not move
  308. * the mapping back due to an elevated page count, we would have to
  309. * block waiting on other references to be dropped.
  310. */
  311. if (mode == MIGRATE_ASYNC && head &&
  312. !buffer_migrate_lock_buffers(head, mode)) {
  313. page_unfreeze_refs(page, expected_count);
  314. spin_unlock_irq(&mapping->tree_lock);
  315. return -EAGAIN;
  316. }
  317. /*
  318. * Now we know that no one else is looking at the page.
  319. */
  320. get_page(newpage); /* add cache reference */
  321. if (PageSwapCache(page)) {
  322. SetPageSwapCache(newpage);
  323. set_page_private(newpage, page_private(page));
  324. }
  325. radix_tree_replace_slot(pslot, newpage);
  326. /*
  327. * Drop cache reference from old page by unfreezing
  328. * to one less reference.
  329. * We know this isn't the last reference.
  330. */
  331. page_unfreeze_refs(page, expected_count - 1);
  332. /*
  333. * If moved to a different zone then also account
  334. * the page for that zone. Other VM counters will be
  335. * taken care of when we establish references to the
  336. * new page and drop references to the old page.
  337. *
  338. * Note that anonymous pages are accounted for
  339. * via NR_FILE_PAGES and NR_ANON_PAGES if they
  340. * are mapped to swap space.
  341. */
  342. __dec_zone_page_state(page, NR_FILE_PAGES);
  343. __inc_zone_page_state(newpage, NR_FILE_PAGES);
  344. if (!PageSwapCache(page) && PageSwapBacked(page)) {
  345. __dec_zone_page_state(page, NR_SHMEM);
  346. __inc_zone_page_state(newpage, NR_SHMEM);
  347. }
  348. spin_unlock_irq(&mapping->tree_lock);
  349. return MIGRATEPAGE_SUCCESS;
  350. }
  351. /*
  352. * The expected number of remaining references is the same as that
  353. * of migrate_page_move_mapping().
  354. */
  355. int migrate_huge_page_move_mapping(struct address_space *mapping,
  356. struct page *newpage, struct page *page)
  357. {
  358. int expected_count;
  359. void **pslot;
  360. if (!mapping) {
  361. if (page_count(page) != 1)
  362. return -EAGAIN;
  363. return MIGRATEPAGE_SUCCESS;
  364. }
  365. spin_lock_irq(&mapping->tree_lock);
  366. pslot = radix_tree_lookup_slot(&mapping->page_tree,
  367. page_index(page));
  368. expected_count = 2 + page_has_private(page);
  369. if (page_count(page) != expected_count ||
  370. radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) {
  371. spin_unlock_irq(&mapping->tree_lock);
  372. return -EAGAIN;
  373. }
  374. if (!page_freeze_refs(page, expected_count)) {
  375. spin_unlock_irq(&mapping->tree_lock);
  376. return -EAGAIN;
  377. }
  378. get_page(newpage);
  379. radix_tree_replace_slot(pslot, newpage);
  380. page_unfreeze_refs(page, expected_count - 1);
  381. spin_unlock_irq(&mapping->tree_lock);
  382. return MIGRATEPAGE_SUCCESS;
  383. }
  384. /*
  385. * Gigantic pages are so large that we do not guarantee that page++ pointer
  386. * arithmetic will work across the entire page. We need something more
  387. * specialized.
  388. */
  389. static void __copy_gigantic_page(struct page *dst, struct page *src,
  390. int nr_pages)
  391. {
  392. int i;
  393. struct page *dst_base = dst;
  394. struct page *src_base = src;
  395. for (i = 0; i < nr_pages; ) {
  396. cond_resched();
  397. copy_highpage(dst, src);
  398. i++;
  399. dst = mem_map_next(dst, dst_base, i);
  400. src = mem_map_next(src, src_base, i);
  401. }
  402. }
  403. static void copy_huge_page(struct page *dst, struct page *src)
  404. {
  405. int i;
  406. int nr_pages;
  407. if (PageHuge(src)) {
  408. /* hugetlbfs page */
  409. struct hstate *h = page_hstate(src);
  410. nr_pages = pages_per_huge_page(h);
  411. if (unlikely(nr_pages > MAX_ORDER_NR_PAGES)) {
  412. __copy_gigantic_page(dst, src, nr_pages);
  413. return;
  414. }
  415. } else {
  416. /* thp page */
  417. BUG_ON(!PageTransHuge(src));
  418. nr_pages = hpage_nr_pages(src);
  419. }
  420. for (i = 0; i < nr_pages; i++) {
  421. cond_resched();
  422. copy_highpage(dst + i, src + i);
  423. }
  424. }
  425. /*
  426. * Copy the page to its new location
  427. */
  428. void migrate_page_copy(struct page *newpage, struct page *page)
  429. {
  430. int cpupid;
  431. if (PageHuge(page) || PageTransHuge(page))
  432. copy_huge_page(newpage, page);
  433. else
  434. copy_highpage(newpage, page);
  435. if (PageError(page))
  436. SetPageError(newpage);
  437. if (PageReferenced(page))
  438. SetPageReferenced(newpage);
  439. if (PageUptodate(page))
  440. SetPageUptodate(newpage);
  441. if (TestClearPageActive(page)) {
  442. VM_BUG_ON(PageUnevictable(page));
  443. SetPageActive(newpage);
  444. } else if (TestClearPageUnevictable(page))
  445. SetPageUnevictable(newpage);
  446. if (PageChecked(page))
  447. SetPageChecked(newpage);
  448. if (PageMappedToDisk(page))
  449. SetPageMappedToDisk(newpage);
  450. if (PageDirty(page)) {
  451. clear_page_dirty_for_io(page);
  452. /*
  453. * Want to mark the page and the radix tree as dirty, and
  454. * redo the accounting that clear_page_dirty_for_io undid,
  455. * but we can't use set_page_dirty because that function
  456. * is actually a signal that all of the page has become dirty.
  457. * Whereas only part of our page may be dirty.
  458. */
  459. if (PageSwapBacked(page))
  460. SetPageDirty(newpage);
  461. else
  462. __set_page_dirty_nobuffers(newpage);
  463. }
  464. /*
  465. * Copy NUMA information to the new page, to prevent over-eager
  466. * future migrations of this same page.
  467. */
  468. cpupid = page_cpupid_xchg_last(page, -1);
  469. page_cpupid_xchg_last(newpage, cpupid);
  470. mlock_migrate_page(newpage, page);
  471. ksm_migrate_page(newpage, page);
  472. /*
  473. * Please do not reorder this without considering how mm/ksm.c's
  474. * get_ksm_page() depends upon ksm_migrate_page() and PageSwapCache().
  475. */
  476. ClearPageSwapCache(page);
  477. ClearPagePrivate(page);
  478. set_page_private(page, 0);
  479. /*
  480. * If any waiters have accumulated on the new page then
  481. * wake them up.
  482. */
  483. if (PageWriteback(newpage))
  484. end_page_writeback(newpage);
  485. }
  486. /************************************************************
  487. * Migration functions
  488. ***********************************************************/
  489. /* Always fail migration. Used for mappings that are not movable */
  490. int fail_migrate_page(struct address_space *mapping,
  491. struct page *newpage, struct page *page)
  492. {
  493. return -EIO;
  494. }
  495. EXPORT_SYMBOL(fail_migrate_page);
  496. /*
  497. * Common logic to directly migrate a single page suitable for
  498. * pages that do not use PagePrivate/PagePrivate2.
  499. *
  500. * Pages are locked upon entry and exit.
  501. */
  502. int migrate_page(struct address_space *mapping,
  503. struct page *newpage, struct page *page,
  504. enum migrate_mode mode)
  505. {
  506. int rc;
  507. BUG_ON(PageWriteback(page)); /* Writeback must be complete */
  508. rc = migrate_page_move_mapping(mapping, newpage, page, NULL, mode);
  509. if (rc != MIGRATEPAGE_SUCCESS)
  510. return rc;
  511. migrate_page_copy(newpage, page);
  512. return MIGRATEPAGE_SUCCESS;
  513. }
  514. EXPORT_SYMBOL(migrate_page);
  515. #ifdef CONFIG_BLOCK
  516. /*
  517. * Migration function for pages with buffers. This function can only be used
  518. * if the underlying filesystem guarantees that no other references to "page"
  519. * exist.
  520. */
  521. int buffer_migrate_page(struct address_space *mapping,
  522. struct page *newpage, struct page *page, enum migrate_mode mode)
  523. {
  524. struct buffer_head *bh, *head;
  525. int rc;
  526. if (!page_has_buffers(page))
  527. return migrate_page(mapping, newpage, page, mode);
  528. head = page_buffers(page);
  529. rc = migrate_page_move_mapping(mapping, newpage, page, head, mode);
  530. if (rc != MIGRATEPAGE_SUCCESS)
  531. return rc;
  532. /*
  533. * In the async case, migrate_page_move_mapping locked the buffers
  534. * with an IRQ-safe spinlock held. In the sync case, the buffers
  535. * need to be locked now
  536. */
  537. if (mode != MIGRATE_ASYNC)
  538. BUG_ON(!buffer_migrate_lock_buffers(head, mode));
  539. ClearPagePrivate(page);
  540. set_page_private(newpage, page_private(page));
  541. set_page_private(page, 0);
  542. put_page(page);
  543. get_page(newpage);
  544. bh = head;
  545. do {
  546. set_bh_page(bh, newpage, bh_offset(bh));
  547. bh = bh->b_this_page;
  548. } while (bh != head);
  549. SetPagePrivate(newpage);
  550. migrate_page_copy(newpage, page);
  551. bh = head;
  552. do {
  553. unlock_buffer(bh);
  554. put_bh(bh);
  555. bh = bh->b_this_page;
  556. } while (bh != head);
  557. return MIGRATEPAGE_SUCCESS;
  558. }
  559. EXPORT_SYMBOL(buffer_migrate_page);
  560. #endif
  561. /*
  562. * Writeback a page to clean the dirty state
  563. */
  564. static int writeout(struct address_space *mapping, struct page *page)
  565. {
  566. struct writeback_control wbc = {
  567. .sync_mode = WB_SYNC_NONE,
  568. .nr_to_write = 1,
  569. .range_start = 0,
  570. .range_end = LLONG_MAX,
  571. .for_reclaim = 1
  572. };
  573. int rc;
  574. if (!mapping->a_ops->writepage)
  575. /* No write method for the address space */
  576. return -EINVAL;
  577. if (!clear_page_dirty_for_io(page))
  578. /* Someone else already triggered a write */
  579. return -EAGAIN;
  580. /*
  581. * A dirty page may imply that the underlying filesystem has
  582. * the page on some queue. So the page must be clean for
  583. * migration. Writeout may mean we loose the lock and the
  584. * page state is no longer what we checked for earlier.
  585. * At this point we know that the migration attempt cannot
  586. * be successful.
  587. */
  588. remove_migration_ptes(page, page);
  589. rc = mapping->a_ops->writepage(page, &wbc);
  590. if (rc != AOP_WRITEPAGE_ACTIVATE)
  591. /* unlocked. Relock */
  592. lock_page(page);
  593. return (rc < 0) ? -EIO : -EAGAIN;
  594. }
  595. /*
  596. * Default handling if a filesystem does not provide a migration function.
  597. */
  598. static int fallback_migrate_page(struct address_space *mapping,
  599. struct page *newpage, struct page *page, enum migrate_mode mode)
  600. {
  601. if (PageDirty(page)) {
  602. /* Only writeback pages in full synchronous migration */
  603. if (mode != MIGRATE_SYNC)
  604. return -EBUSY;
  605. return writeout(mapping, page);
  606. }
  607. /*
  608. * Buffers may be managed in a filesystem specific way.
  609. * We must have no buffers or drop them.
  610. */
  611. if (page_has_private(page) &&
  612. !try_to_release_page(page, GFP_KERNEL))
  613. return -EAGAIN;
  614. return migrate_page(mapping, newpage, page, mode);
  615. }
  616. /*
  617. * Move a page to a newly allocated page
  618. * The page is locked and all ptes have been successfully removed.
  619. *
  620. * The new page will have replaced the old page if this function
  621. * is successful.
  622. *
  623. * Return value:
  624. * < 0 - error code
  625. * MIGRATEPAGE_SUCCESS - success
  626. */
  627. static int move_to_new_page(struct page *newpage, struct page *page,
  628. int remap_swapcache, enum migrate_mode mode)
  629. {
  630. struct address_space *mapping;
  631. int rc;
  632. /*
  633. * Block others from accessing the page when we get around to
  634. * establishing additional references. We are the only one
  635. * holding a reference to the new page at this point.
  636. */
  637. if (!trylock_page(newpage))
  638. BUG();
  639. /* Prepare mapping for the new page.*/
  640. newpage->index = page->index;
  641. newpage->mapping = page->mapping;
  642. if (PageSwapBacked(page))
  643. SetPageSwapBacked(newpage);
  644. mapping = page_mapping(page);
  645. if (!mapping)
  646. rc = migrate_page(mapping, newpage, page, mode);
  647. else if (mapping->a_ops->migratepage)
  648. /*
  649. * Most pages have a mapping and most filesystems provide a
  650. * migratepage callback. Anonymous pages are part of swap
  651. * space which also has its own migratepage callback. This
  652. * is the most common path for page migration.
  653. */
  654. rc = mapping->a_ops->migratepage(mapping,
  655. newpage, page, mode);
  656. else
  657. rc = fallback_migrate_page(mapping, newpage, page, mode);
  658. if (rc != MIGRATEPAGE_SUCCESS) {
  659. newpage->mapping = NULL;
  660. } else {
  661. if (remap_swapcache)
  662. remove_migration_ptes(page, newpage);
  663. page->mapping = NULL;
  664. }
  665. unlock_page(newpage);
  666. return rc;
  667. }
  668. static int __unmap_and_move(struct page *page, struct page *newpage,
  669. int force, enum migrate_mode mode)
  670. {
  671. int rc = -EAGAIN;
  672. int remap_swapcache = 1;
  673. struct mem_cgroup *mem;
  674. struct anon_vma *anon_vma = NULL;
  675. if (!trylock_page(page)) {
  676. if (!force || mode == MIGRATE_ASYNC)
  677. goto out;
  678. /*
  679. * It's not safe for direct compaction to call lock_page.
  680. * For example, during page readahead pages are added locked
  681. * to the LRU. Later, when the IO completes the pages are
  682. * marked uptodate and unlocked. However, the queueing
  683. * could be merging multiple pages for one bio (e.g.
  684. * mpage_readpages). If an allocation happens for the
  685. * second or third page, the process can end up locking
  686. * the same page twice and deadlocking. Rather than
  687. * trying to be clever about what pages can be locked,
  688. * avoid the use of lock_page for direct compaction
  689. * altogether.
  690. */
  691. if (current->flags & PF_MEMALLOC)
  692. goto out;
  693. lock_page(page);
  694. }
  695. /* charge against new page */
  696. mem_cgroup_prepare_migration(page, newpage, &mem);
  697. if (PageWriteback(page)) {
  698. /*
  699. * Only in the case of a full synchronous migration is it
  700. * necessary to wait for PageWriteback. In the async case,
  701. * the retry loop is too short and in the sync-light case,
  702. * the overhead of stalling is too much
  703. */
  704. if (mode != MIGRATE_SYNC) {
  705. rc = -EBUSY;
  706. goto uncharge;
  707. }
  708. if (!force)
  709. goto uncharge;
  710. wait_on_page_writeback(page);
  711. }
  712. /*
  713. * By try_to_unmap(), page->mapcount goes down to 0 here. In this case,
  714. * we cannot notice that anon_vma is freed while we migrates a page.
  715. * This get_anon_vma() delays freeing anon_vma pointer until the end
  716. * of migration. File cache pages are no problem because of page_lock()
  717. * File Caches may use write_page() or lock_page() in migration, then,
  718. * just care Anon page here.
  719. */
  720. if (PageAnon(page) && !PageKsm(page)) {
  721. /*
  722. * Only page_lock_anon_vma_read() understands the subtleties of
  723. * getting a hold on an anon_vma from outside one of its mms.
  724. */
  725. anon_vma = page_get_anon_vma(page);
  726. if (anon_vma) {
  727. /*
  728. * Anon page
  729. */
  730. } else if (PageSwapCache(page)) {
  731. /*
  732. * We cannot be sure that the anon_vma of an unmapped
  733. * swapcache page is safe to use because we don't
  734. * know in advance if the VMA that this page belonged
  735. * to still exists. If the VMA and others sharing the
  736. * data have been freed, then the anon_vma could
  737. * already be invalid.
  738. *
  739. * To avoid this possibility, swapcache pages get
  740. * migrated but are not remapped when migration
  741. * completes
  742. */
  743. remap_swapcache = 0;
  744. } else {
  745. goto uncharge;
  746. }
  747. }
  748. if (unlikely(balloon_page_movable(page))) {
  749. /*
  750. * A ballooned page does not need any special attention from
  751. * physical to virtual reverse mapping procedures.
  752. * Skip any attempt to unmap PTEs or to remap swap cache,
  753. * in order to avoid burning cycles at rmap level, and perform
  754. * the page migration right away (proteced by page lock).
  755. */
  756. rc = balloon_page_migrate(newpage, page, mode);
  757. goto uncharge;
  758. }
  759. /*
  760. * Corner case handling:
  761. * 1. When a new swap-cache page is read into, it is added to the LRU
  762. * and treated as swapcache but it has no rmap yet.
  763. * Calling try_to_unmap() against a page->mapping==NULL page will
  764. * trigger a BUG. So handle it here.
  765. * 2. An orphaned page (see truncate_complete_page) might have
  766. * fs-private metadata. The page can be picked up due to memory
  767. * offlining. Everywhere else except page reclaim, the page is
  768. * invisible to the vm, so the page can not be migrated. So try to
  769. * free the metadata, so the page can be freed.
  770. */
  771. if (!page->mapping) {
  772. VM_BUG_ON(PageAnon(page));
  773. if (page_has_private(page)) {
  774. try_to_free_buffers(page);
  775. goto uncharge;
  776. }
  777. goto skip_unmap;
  778. }
  779. /* Establish migration ptes or remove ptes */
  780. try_to_unmap(page, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
  781. skip_unmap:
  782. if (!page_mapped(page))
  783. rc = move_to_new_page(newpage, page, remap_swapcache, mode);
  784. if (rc && remap_swapcache)
  785. remove_migration_ptes(page, page);
  786. /* Drop an anon_vma reference if we took one */
  787. if (anon_vma)
  788. put_anon_vma(anon_vma);
  789. uncharge:
  790. mem_cgroup_end_migration(mem, page, newpage,
  791. (rc == MIGRATEPAGE_SUCCESS ||
  792. rc == MIGRATEPAGE_BALLOON_SUCCESS));
  793. unlock_page(page);
  794. out:
  795. return rc;
  796. }
  797. /*
  798. * Obtain the lock on page, remove all ptes and migrate the page
  799. * to the newly allocated page in newpage.
  800. */
  801. static int unmap_and_move(new_page_t get_new_page, unsigned long private,
  802. struct page *page, int force, enum migrate_mode mode)
  803. {
  804. int rc = 0;
  805. int *result = NULL;
  806. struct page *newpage = get_new_page(page, private, &result);
  807. if (!newpage)
  808. return -ENOMEM;
  809. if (page_count(page) == 1) {
  810. /* page was freed from under us. So we are done. */
  811. goto out;
  812. }
  813. if (unlikely(PageTransHuge(page)))
  814. if (unlikely(split_huge_page(page)))
  815. goto out;
  816. rc = __unmap_and_move(page, newpage, force, mode);
  817. if (unlikely(rc == MIGRATEPAGE_BALLOON_SUCCESS)) {
  818. /*
  819. * A ballooned page has been migrated already.
  820. * Now, it's the time to wrap-up counters,
  821. * handle the page back to Buddy and return.
  822. */
  823. dec_zone_page_state(page, NR_ISOLATED_ANON +
  824. page_is_file_cache(page));
  825. balloon_page_free(page);
  826. return MIGRATEPAGE_SUCCESS;
  827. }
  828. out:
  829. if (rc != -EAGAIN) {
  830. /*
  831. * A page that has been migrated has all references
  832. * removed and will be freed. A page that has not been
  833. * migrated will have kepts its references and be
  834. * restored.
  835. */
  836. list_del(&page->lru);
  837. dec_zone_page_state(page, NR_ISOLATED_ANON +
  838. page_is_file_cache(page));
  839. putback_lru_page(page);
  840. }
  841. /*
  842. * Move the new page to the LRU. If migration was not successful
  843. * then this will free the page.
  844. */
  845. putback_lru_page(newpage);
  846. if (result) {
  847. if (rc)
  848. *result = rc;
  849. else
  850. *result = page_to_nid(newpage);
  851. }
  852. return rc;
  853. }
  854. /*
  855. * Counterpart of unmap_and_move_page() for hugepage migration.
  856. *
  857. * This function doesn't wait the completion of hugepage I/O
  858. * because there is no race between I/O and migration for hugepage.
  859. * Note that currently hugepage I/O occurs only in direct I/O
  860. * where no lock is held and PG_writeback is irrelevant,
  861. * and writeback status of all subpages are counted in the reference
  862. * count of the head page (i.e. if all subpages of a 2MB hugepage are
  863. * under direct I/O, the reference of the head page is 512 and a bit more.)
  864. * This means that when we try to migrate hugepage whose subpages are
  865. * doing direct I/O, some references remain after try_to_unmap() and
  866. * hugepage migration fails without data corruption.
  867. *
  868. * There is also no race when direct I/O is issued on the page under migration,
  869. * because then pte is replaced with migration swap entry and direct I/O code
  870. * will wait in the page fault for migration to complete.
  871. */
  872. static int unmap_and_move_huge_page(new_page_t get_new_page,
  873. unsigned long private, struct page *hpage,
  874. int force, enum migrate_mode mode)
  875. {
  876. int rc = 0;
  877. int *result = NULL;
  878. struct page *new_hpage = get_new_page(hpage, private, &result);
  879. struct anon_vma *anon_vma = NULL;
  880. /*
  881. * Movability of hugepages depends on architectures and hugepage size.
  882. * This check is necessary because some callers of hugepage migration
  883. * like soft offline and memory hotremove don't walk through page
  884. * tables or check whether the hugepage is pmd-based or not before
  885. * kicking migration.
  886. */
  887. if (!hugepage_migration_support(page_hstate(hpage)))
  888. return -ENOSYS;
  889. if (!new_hpage)
  890. return -ENOMEM;
  891. rc = -EAGAIN;
  892. if (!trylock_page(hpage)) {
  893. if (!force || mode != MIGRATE_SYNC)
  894. goto out;
  895. lock_page(hpage);
  896. }
  897. if (PageAnon(hpage))
  898. anon_vma = page_get_anon_vma(hpage);
  899. try_to_unmap(hpage, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
  900. if (!page_mapped(hpage))
  901. rc = move_to_new_page(new_hpage, hpage, 1, mode);
  902. if (rc)
  903. remove_migration_ptes(hpage, hpage);
  904. if (anon_vma)
  905. put_anon_vma(anon_vma);
  906. if (!rc)
  907. hugetlb_cgroup_migrate(hpage, new_hpage);
  908. unlock_page(hpage);
  909. out:
  910. if (rc != -EAGAIN)
  911. putback_active_hugepage(hpage);
  912. put_page(new_hpage);
  913. if (result) {
  914. if (rc)
  915. *result = rc;
  916. else
  917. *result = page_to_nid(new_hpage);
  918. }
  919. return rc;
  920. }
  921. /*
  922. * migrate_pages - migrate the pages specified in a list, to the free pages
  923. * supplied as the target for the page migration
  924. *
  925. * @from: The list of pages to be migrated.
  926. * @get_new_page: The function used to allocate free pages to be used
  927. * as the target of the page migration.
  928. * @private: Private data to be passed on to get_new_page()
  929. * @mode: The migration mode that specifies the constraints for
  930. * page migration, if any.
  931. * @reason: The reason for page migration.
  932. *
  933. * The function returns after 10 attempts or if no pages are movable any more
  934. * because the list has become empty or no retryable pages exist any more.
  935. * The caller should call putback_lru_pages() to return pages to the LRU
  936. * or free list only if ret != 0.
  937. *
  938. * Returns the number of pages that were not migrated, or an error code.
  939. */
  940. int migrate_pages(struct list_head *from, new_page_t get_new_page,
  941. unsigned long private, enum migrate_mode mode, int reason)
  942. {
  943. int retry = 1;
  944. int nr_failed = 0;
  945. int nr_succeeded = 0;
  946. int pass = 0;
  947. struct page *page;
  948. struct page *page2;
  949. int swapwrite = current->flags & PF_SWAPWRITE;
  950. int rc;
  951. if (!swapwrite)
  952. current->flags |= PF_SWAPWRITE;
  953. for(pass = 0; pass < 10 && retry; pass++) {
  954. retry = 0;
  955. list_for_each_entry_safe(page, page2, from, lru) {
  956. cond_resched();
  957. if (PageHuge(page))
  958. rc = unmap_and_move_huge_page(get_new_page,
  959. private, page, pass > 2, mode);
  960. else
  961. rc = unmap_and_move(get_new_page, private,
  962. page, pass > 2, mode);
  963. switch(rc) {
  964. case -ENOMEM:
  965. goto out;
  966. case -EAGAIN:
  967. retry++;
  968. break;
  969. case MIGRATEPAGE_SUCCESS:
  970. nr_succeeded++;
  971. break;
  972. default:
  973. /* Permanent failure */
  974. nr_failed++;
  975. break;
  976. }
  977. }
  978. }
  979. rc = nr_failed + retry;
  980. out:
  981. if (nr_succeeded)
  982. count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded);
  983. if (nr_failed)
  984. count_vm_events(PGMIGRATE_FAIL, nr_failed);
  985. trace_mm_migrate_pages(nr_succeeded, nr_failed, mode, reason);
  986. if (!swapwrite)
  987. current->flags &= ~PF_SWAPWRITE;
  988. return rc;
  989. }
  990. #ifdef CONFIG_NUMA
  991. /*
  992. * Move a list of individual pages
  993. */
  994. struct page_to_node {
  995. unsigned long addr;
  996. struct page *page;
  997. int node;
  998. int status;
  999. };
  1000. static struct page *new_page_node(struct page *p, unsigned long private,
  1001. int **result)
  1002. {
  1003. struct page_to_node *pm = (struct page_to_node *)private;
  1004. while (pm->node != MAX_NUMNODES && pm->page != p)
  1005. pm++;
  1006. if (pm->node == MAX_NUMNODES)
  1007. return NULL;
  1008. *result = &pm->status;
  1009. if (PageHuge(p))
  1010. return alloc_huge_page_node(page_hstate(compound_head(p)),
  1011. pm->node);
  1012. else
  1013. return alloc_pages_exact_node(pm->node,
  1014. GFP_HIGHUSER_MOVABLE | GFP_THISNODE, 0);
  1015. }
  1016. /*
  1017. * Move a set of pages as indicated in the pm array. The addr
  1018. * field must be set to the virtual address of the page to be moved
  1019. * and the node number must contain a valid target node.
  1020. * The pm array ends with node = MAX_NUMNODES.
  1021. */
  1022. static int do_move_page_to_node_array(struct mm_struct *mm,
  1023. struct page_to_node *pm,
  1024. int migrate_all)
  1025. {
  1026. int err;
  1027. struct page_to_node *pp;
  1028. LIST_HEAD(pagelist);
  1029. down_read(&mm->mmap_sem);
  1030. /*
  1031. * Build a list of pages to migrate
  1032. */
  1033. for (pp = pm; pp->node != MAX_NUMNODES; pp++) {
  1034. struct vm_area_struct *vma;
  1035. struct page *page;
  1036. err = -EFAULT;
  1037. vma = find_vma(mm, pp->addr);
  1038. if (!vma || pp->addr < vma->vm_start || !vma_migratable(vma))
  1039. goto set_status;
  1040. page = follow_page(vma, pp->addr, FOLL_GET|FOLL_SPLIT);
  1041. err = PTR_ERR(page);
  1042. if (IS_ERR(page))
  1043. goto set_status;
  1044. err = -ENOENT;
  1045. if (!page)
  1046. goto set_status;
  1047. /* Use PageReserved to check for zero page */
  1048. if (PageReserved(page))
  1049. goto put_and_set;
  1050. pp->page = page;
  1051. err = page_to_nid(page);
  1052. if (err == pp->node)
  1053. /*
  1054. * Node already in the right place
  1055. */
  1056. goto put_and_set;
  1057. err = -EACCES;
  1058. if (page_mapcount(page) > 1 &&
  1059. !migrate_all)
  1060. goto put_and_set;
  1061. if (PageHuge(page)) {
  1062. isolate_huge_page(page, &pagelist);
  1063. goto put_and_set;
  1064. }
  1065. err = isolate_lru_page(page);
  1066. if (!err) {
  1067. list_add_tail(&page->lru, &pagelist);
  1068. inc_zone_page_state(page, NR_ISOLATED_ANON +
  1069. page_is_file_cache(page));
  1070. }
  1071. put_and_set:
  1072. /*
  1073. * Either remove the duplicate refcount from
  1074. * isolate_lru_page() or drop the page ref if it was
  1075. * not isolated.
  1076. */
  1077. put_page(page);
  1078. set_status:
  1079. pp->status = err;
  1080. }
  1081. err = 0;
  1082. if (!list_empty(&pagelist)) {
  1083. err = migrate_pages(&pagelist, new_page_node,
  1084. (unsigned long)pm, MIGRATE_SYNC, MR_SYSCALL);
  1085. if (err)
  1086. putback_movable_pages(&pagelist);
  1087. }
  1088. up_read(&mm->mmap_sem);
  1089. return err;
  1090. }
  1091. /*
  1092. * Migrate an array of page address onto an array of nodes and fill
  1093. * the corresponding array of status.
  1094. */
  1095. static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes,
  1096. unsigned long nr_pages,
  1097. const void __user * __user *pages,
  1098. const int __user *nodes,
  1099. int __user *status, int flags)
  1100. {
  1101. struct page_to_node *pm;
  1102. unsigned long chunk_nr_pages;
  1103. unsigned long chunk_start;
  1104. int err;
  1105. err = -ENOMEM;
  1106. pm = (struct page_to_node *)__get_free_page(GFP_KERNEL);
  1107. if (!pm)
  1108. goto out;
  1109. migrate_prep();
  1110. /*
  1111. * Store a chunk of page_to_node array in a page,
  1112. * but keep the last one as a marker
  1113. */
  1114. chunk_nr_pages = (PAGE_SIZE / sizeof(struct page_to_node)) - 1;
  1115. for (chunk_start = 0;
  1116. chunk_start < nr_pages;
  1117. chunk_start += chunk_nr_pages) {
  1118. int j;
  1119. if (chunk_start + chunk_nr_pages > nr_pages)
  1120. chunk_nr_pages = nr_pages - chunk_start;
  1121. /* fill the chunk pm with addrs and nodes from user-space */
  1122. for (j = 0; j < chunk_nr_pages; j++) {
  1123. const void __user *p;
  1124. int node;
  1125. err = -EFAULT;
  1126. if (get_user(p, pages + j + chunk_start))
  1127. goto out_pm;
  1128. pm[j].addr = (unsigned long) p;
  1129. if (get_user(node, nodes + j + chunk_start))
  1130. goto out_pm;
  1131. err = -ENODEV;
  1132. if (node < 0 || node >= MAX_NUMNODES)
  1133. goto out_pm;
  1134. if (!node_state(node, N_MEMORY))
  1135. goto out_pm;
  1136. err = -EACCES;
  1137. if (!node_isset(node, task_nodes))
  1138. goto out_pm;
  1139. pm[j].node = node;
  1140. }
  1141. /* End marker for this chunk */
  1142. pm[chunk_nr_pages].node = MAX_NUMNODES;
  1143. /* Migrate this chunk */
  1144. err = do_move_page_to_node_array(mm, pm,
  1145. flags & MPOL_MF_MOVE_ALL);
  1146. if (err < 0)
  1147. goto out_pm;
  1148. /* Return status information */
  1149. for (j = 0; j < chunk_nr_pages; j++)
  1150. if (put_user(pm[j].status, status + j + chunk_start)) {
  1151. err = -EFAULT;
  1152. goto out_pm;
  1153. }
  1154. }
  1155. err = 0;
  1156. out_pm:
  1157. free_page((unsigned long)pm);
  1158. out:
  1159. return err;
  1160. }
  1161. /*
  1162. * Determine the nodes of an array of pages and store it in an array of status.
  1163. */
  1164. static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages,
  1165. const void __user **pages, int *status)
  1166. {
  1167. unsigned long i;
  1168. down_read(&mm->mmap_sem);
  1169. for (i = 0; i < nr_pages; i++) {
  1170. unsigned long addr = (unsigned long)(*pages);
  1171. struct vm_area_struct *vma;
  1172. struct page *page;
  1173. int err = -EFAULT;
  1174. vma = find_vma(mm, addr);
  1175. if (!vma || addr < vma->vm_start)
  1176. goto set_status;
  1177. page = follow_page(vma, addr, 0);
  1178. err = PTR_ERR(page);
  1179. if (IS_ERR(page))
  1180. goto set_status;
  1181. err = -ENOENT;
  1182. /* Use PageReserved to check for zero page */
  1183. if (!page || PageReserved(page))
  1184. goto set_status;
  1185. err = page_to_nid(page);
  1186. set_status:
  1187. *status = err;
  1188. pages++;
  1189. status++;
  1190. }
  1191. up_read(&mm->mmap_sem);
  1192. }
  1193. /*
  1194. * Determine the nodes of a user array of pages and store it in
  1195. * a user array of status.
  1196. */
  1197. static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages,
  1198. const void __user * __user *pages,
  1199. int __user *status)
  1200. {
  1201. #define DO_PAGES_STAT_CHUNK_NR 16
  1202. const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR];
  1203. int chunk_status[DO_PAGES_STAT_CHUNK_NR];
  1204. while (nr_pages) {
  1205. unsigned long chunk_nr;
  1206. chunk_nr = nr_pages;
  1207. if (chunk_nr > DO_PAGES_STAT_CHUNK_NR)
  1208. chunk_nr = DO_PAGES_STAT_CHUNK_NR;
  1209. if (copy_from_user(chunk_pages, pages, chunk_nr * sizeof(*chunk_pages)))
  1210. break;
  1211. do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status);
  1212. if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status)))
  1213. break;
  1214. pages += chunk_nr;
  1215. status += chunk_nr;
  1216. nr_pages -= chunk_nr;
  1217. }
  1218. return nr_pages ? -EFAULT : 0;
  1219. }
  1220. /*
  1221. * Move a list of pages in the address space of the currently executing
  1222. * process.
  1223. */
  1224. SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages,
  1225. const void __user * __user *, pages,
  1226. const int __user *, nodes,
  1227. int __user *, status, int, flags)
  1228. {
  1229. const struct cred *cred = current_cred(), *tcred;
  1230. struct task_struct *task;
  1231. struct mm_struct *mm;
  1232. int err;
  1233. nodemask_t task_nodes;
  1234. /* Check flags */
  1235. if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL))
  1236. return -EINVAL;
  1237. if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
  1238. return -EPERM;
  1239. /* Find the mm_struct */
  1240. rcu_read_lock();
  1241. task = pid ? find_task_by_vpid(pid) : current;
  1242. if (!task) {
  1243. rcu_read_unlock();
  1244. return -ESRCH;
  1245. }
  1246. get_task_struct(task);
  1247. /*
  1248. * Check if this process has the right to modify the specified
  1249. * process. The right exists if the process has administrative
  1250. * capabilities, superuser privileges or the same
  1251. * userid as the target process.
  1252. */
  1253. tcred = __task_cred(task);
  1254. if (!uid_eq(cred->euid, tcred->suid) && !uid_eq(cred->euid, tcred->uid) &&
  1255. !uid_eq(cred->uid, tcred->suid) && !uid_eq(cred->uid, tcred->uid) &&
  1256. !capable(CAP_SYS_NICE)) {
  1257. rcu_read_unlock();
  1258. err = -EPERM;
  1259. goto out;
  1260. }
  1261. rcu_read_unlock();
  1262. err = security_task_movememory(task);
  1263. if (err)
  1264. goto out;
  1265. task_nodes = cpuset_mems_allowed(task);
  1266. mm = get_task_mm(task);
  1267. put_task_struct(task);
  1268. if (!mm)
  1269. return -EINVAL;
  1270. if (nodes)
  1271. err = do_pages_move(mm, task_nodes, nr_pages, pages,
  1272. nodes, status, flags);
  1273. else
  1274. err = do_pages_stat(mm, nr_pages, pages, status);
  1275. mmput(mm);
  1276. return err;
  1277. out:
  1278. put_task_struct(task);
  1279. return err;
  1280. }
  1281. /*
  1282. * Call migration functions in the vma_ops that may prepare
  1283. * memory in a vm for migration. migration functions may perform
  1284. * the migration for vmas that do not have an underlying page struct.
  1285. */
  1286. int migrate_vmas(struct mm_struct *mm, const nodemask_t *to,
  1287. const nodemask_t *from, unsigned long flags)
  1288. {
  1289. struct vm_area_struct *vma;
  1290. int err = 0;
  1291. for (vma = mm->mmap; vma && !err; vma = vma->vm_next) {
  1292. if (vma->vm_ops && vma->vm_ops->migrate) {
  1293. err = vma->vm_ops->migrate(vma, to, from, flags);
  1294. if (err)
  1295. break;
  1296. }
  1297. }
  1298. return err;
  1299. }
  1300. #ifdef CONFIG_NUMA_BALANCING
  1301. /*
  1302. * Returns true if this is a safe migration target node for misplaced NUMA
  1303. * pages. Currently it only checks the watermarks which crude
  1304. */
  1305. static bool migrate_balanced_pgdat(struct pglist_data *pgdat,
  1306. unsigned long nr_migrate_pages)
  1307. {
  1308. int z;
  1309. for (z = pgdat->nr_zones - 1; z >= 0; z--) {
  1310. struct zone *zone = pgdat->node_zones + z;
  1311. if (!populated_zone(zone))
  1312. continue;
  1313. if (!zone_reclaimable(zone))
  1314. continue;
  1315. /* Avoid waking kswapd by allocating pages_to_migrate pages. */
  1316. if (!zone_watermark_ok(zone, 0,
  1317. high_wmark_pages(zone) +
  1318. nr_migrate_pages,
  1319. 0, 0))
  1320. continue;
  1321. return true;
  1322. }
  1323. return false;
  1324. }
  1325. static struct page *alloc_misplaced_dst_page(struct page *page,
  1326. unsigned long data,
  1327. int **result)
  1328. {
  1329. int nid = (int) data;
  1330. struct page *newpage;
  1331. newpage = alloc_pages_exact_node(nid,
  1332. (GFP_HIGHUSER_MOVABLE | GFP_THISNODE |
  1333. __GFP_NOMEMALLOC | __GFP_NORETRY |
  1334. __GFP_NOWARN) &
  1335. ~GFP_IOFS, 0);
  1336. if (newpage)
  1337. page_cpupid_xchg_last(newpage, page_cpupid_last(page));
  1338. return newpage;
  1339. }
  1340. /*
  1341. * page migration rate limiting control.
  1342. * Do not migrate more than @pages_to_migrate in a @migrate_interval_millisecs
  1343. * window of time. Default here says do not migrate more than 1280M per second.
  1344. * If a node is rate-limited then PTE NUMA updates are also rate-limited. However
  1345. * as it is faults that reset the window, pte updates will happen unconditionally
  1346. * if there has not been a fault since @pteupdate_interval_millisecs after the
  1347. * throttle window closed.
  1348. */
  1349. static unsigned int migrate_interval_millisecs __read_mostly = 100;
  1350. static unsigned int pteupdate_interval_millisecs __read_mostly = 1000;
  1351. static unsigned int ratelimit_pages __read_mostly = 128 << (20 - PAGE_SHIFT);
  1352. /* Returns true if NUMA migration is currently rate limited */
  1353. bool migrate_ratelimited(int node)
  1354. {
  1355. pg_data_t *pgdat = NODE_DATA(node);
  1356. if (time_after(jiffies, pgdat->numabalancing_migrate_next_window +
  1357. msecs_to_jiffies(pteupdate_interval_millisecs)))
  1358. return false;
  1359. if (pgdat->numabalancing_migrate_nr_pages < ratelimit_pages)
  1360. return false;
  1361. return true;
  1362. }
  1363. /* Returns true if the node is migrate rate-limited after the update */
  1364. bool numamigrate_update_ratelimit(pg_data_t *pgdat, unsigned long nr_pages)
  1365. {
  1366. bool rate_limited = false;
  1367. /*
  1368. * Rate-limit the amount of data that is being migrated to a node.
  1369. * Optimal placement is no good if the memory bus is saturated and
  1370. * all the time is being spent migrating!
  1371. */
  1372. spin_lock(&pgdat->numabalancing_migrate_lock);
  1373. if (time_after(jiffies, pgdat->numabalancing_migrate_next_window)) {
  1374. pgdat->numabalancing_migrate_nr_pages = 0;
  1375. pgdat->numabalancing_migrate_next_window = jiffies +
  1376. msecs_to_jiffies(migrate_interval_millisecs);
  1377. }
  1378. if (pgdat->numabalancing_migrate_nr_pages > ratelimit_pages)
  1379. rate_limited = true;
  1380. else
  1381. pgdat->numabalancing_migrate_nr_pages += nr_pages;
  1382. spin_unlock(&pgdat->numabalancing_migrate_lock);
  1383. return rate_limited;
  1384. }
  1385. int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page)
  1386. {
  1387. int page_lru;
  1388. VM_BUG_ON(compound_order(page) && !PageTransHuge(page));
  1389. /* Avoid migrating to a node that is nearly full */
  1390. if (!migrate_balanced_pgdat(pgdat, 1UL << compound_order(page)))
  1391. return 0;
  1392. if (isolate_lru_page(page))
  1393. return 0;
  1394. /*
  1395. * migrate_misplaced_transhuge_page() skips page migration's usual
  1396. * check on page_count(), so we must do it here, now that the page
  1397. * has been isolated: a GUP pin, or any other pin, prevents migration.
  1398. * The expected page count is 3: 1 for page's mapcount and 1 for the
  1399. * caller's pin and 1 for the reference taken by isolate_lru_page().
  1400. */
  1401. if (PageTransHuge(page) && page_count(page) != 3) {
  1402. putback_lru_page(page);
  1403. return 0;
  1404. }
  1405. page_lru = page_is_file_cache(page);
  1406. mod_zone_page_state(page_zone(page), NR_ISOLATED_ANON + page_lru,
  1407. hpage_nr_pages(page));
  1408. /*
  1409. * Isolating the page has taken another reference, so the
  1410. * caller's reference can be safely dropped without the page
  1411. * disappearing underneath us during migration.
  1412. */
  1413. put_page(page);
  1414. return 1;
  1415. }
  1416. /*
  1417. * Attempt to migrate a misplaced page to the specified destination
  1418. * node. Caller is expected to have an elevated reference count on
  1419. * the page that will be dropped by this function before returning.
  1420. */
  1421. int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma,
  1422. int node)
  1423. {
  1424. pg_data_t *pgdat = NODE_DATA(node);
  1425. int isolated;
  1426. int nr_remaining;
  1427. LIST_HEAD(migratepages);
  1428. /*
  1429. * Don't migrate file pages that are mapped in multiple processes
  1430. * with execute permissions as they are probably shared libraries.
  1431. */
  1432. if (page_mapcount(page) != 1 && page_is_file_cache(page) &&
  1433. (vma->vm_flags & VM_EXEC))
  1434. goto out;
  1435. /*
  1436. * Rate-limit the amount of data that is being migrated to a node.
  1437. * Optimal placement is no good if the memory bus is saturated and
  1438. * all the time is being spent migrating!
  1439. */
  1440. if (numamigrate_update_ratelimit(pgdat, 1))
  1441. goto out;
  1442. isolated = numamigrate_isolate_page(pgdat, page);
  1443. if (!isolated)
  1444. goto out;
  1445. list_add(&page->lru, &migratepages);
  1446. nr_remaining = migrate_pages(&migratepages, alloc_misplaced_dst_page,
  1447. node, MIGRATE_ASYNC, MR_NUMA_MISPLACED);
  1448. if (nr_remaining) {
  1449. putback_lru_pages(&migratepages);
  1450. isolated = 0;
  1451. } else
  1452. count_vm_numa_event(NUMA_PAGE_MIGRATE);
  1453. BUG_ON(!list_empty(&migratepages));
  1454. return isolated;
  1455. out:
  1456. put_page(page);
  1457. return 0;
  1458. }
  1459. #endif /* CONFIG_NUMA_BALANCING */
  1460. #if defined(CONFIG_NUMA_BALANCING) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
  1461. /*
  1462. * Migrates a THP to a given target node. page must be locked and is unlocked
  1463. * before returning.
  1464. */
  1465. int migrate_misplaced_transhuge_page(struct mm_struct *mm,
  1466. struct vm_area_struct *vma,
  1467. pmd_t *pmd, pmd_t entry,
  1468. unsigned long address,
  1469. struct page *page, int node)
  1470. {
  1471. spinlock_t *ptl;
  1472. unsigned long haddr = address & HPAGE_PMD_MASK;
  1473. pg_data_t *pgdat = NODE_DATA(node);
  1474. int isolated = 0;
  1475. struct page *new_page = NULL;
  1476. struct mem_cgroup *memcg = NULL;
  1477. int page_lru = page_is_file_cache(page);
  1478. /*
  1479. * Rate-limit the amount of data that is being migrated to a node.
  1480. * Optimal placement is no good if the memory bus is saturated and
  1481. * all the time is being spent migrating!
  1482. */
  1483. if (numamigrate_update_ratelimit(pgdat, HPAGE_PMD_NR))
  1484. goto out_dropref;
  1485. new_page = alloc_pages_node(node,
  1486. (GFP_TRANSHUGE | GFP_THISNODE) & ~__GFP_WAIT, HPAGE_PMD_ORDER);
  1487. if (!new_page)
  1488. goto out_fail;
  1489. page_cpupid_xchg_last(new_page, page_cpupid_last(page));
  1490. isolated = numamigrate_isolate_page(pgdat, page);
  1491. if (!isolated) {
  1492. put_page(new_page);
  1493. goto out_fail;
  1494. }
  1495. /* Prepare a page as a migration target */
  1496. __set_page_locked(new_page);
  1497. SetPageSwapBacked(new_page);
  1498. /* anon mapping, we can simply copy page->mapping to the new page: */
  1499. new_page->mapping = page->mapping;
  1500. new_page->index = page->index;
  1501. migrate_page_copy(new_page, page);
  1502. WARN_ON(PageLRU(new_page));
  1503. /* Recheck the target PMD */
  1504. ptl = pmd_lock(mm, pmd);
  1505. if (unlikely(!pmd_same(*pmd, entry))) {
  1506. spin_unlock(ptl);
  1507. /* Reverse changes made by migrate_page_copy() */
  1508. if (TestClearPageActive(new_page))
  1509. SetPageActive(page);
  1510. if (TestClearPageUnevictable(new_page))
  1511. SetPageUnevictable(page);
  1512. mlock_migrate_page(page, new_page);
  1513. unlock_page(new_page);
  1514. put_page(new_page); /* Free it */
  1515. /* Retake the callers reference and putback on LRU */
  1516. get_page(page);
  1517. putback_lru_page(page);
  1518. mod_zone_page_state(page_zone(page),
  1519. NR_ISOLATED_ANON + page_lru, -HPAGE_PMD_NR);
  1520. goto out_fail;
  1521. }
  1522. /*
  1523. * Traditional migration needs to prepare the memcg charge
  1524. * transaction early to prevent the old page from being
  1525. * uncharged when installing migration entries. Here we can
  1526. * save the potential rollback and start the charge transfer
  1527. * only when migration is already known to end successfully.
  1528. */
  1529. mem_cgroup_prepare_migration(page, new_page, &memcg);
  1530. entry = mk_pmd(new_page, vma->vm_page_prot);
  1531. entry = pmd_mknonnuma(entry);
  1532. entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
  1533. entry = pmd_mkhuge(entry);
  1534. pmdp_clear_flush(vma, haddr, pmd);
  1535. set_pmd_at(mm, haddr, pmd, entry);
  1536. page_add_new_anon_rmap(new_page, vma, haddr);
  1537. update_mmu_cache_pmd(vma, address, &entry);
  1538. page_remove_rmap(page);
  1539. /*
  1540. * Finish the charge transaction under the page table lock to
  1541. * prevent split_huge_page() from dividing up the charge
  1542. * before it's fully transferred to the new page.
  1543. */
  1544. mem_cgroup_end_migration(memcg, page, new_page, true);
  1545. spin_unlock(ptl);
  1546. unlock_page(new_page);
  1547. unlock_page(page);
  1548. put_page(page); /* Drop the rmap reference */
  1549. put_page(page); /* Drop the LRU isolation reference */
  1550. count_vm_events(PGMIGRATE_SUCCESS, HPAGE_PMD_NR);
  1551. count_vm_numa_events(NUMA_PAGE_MIGRATE, HPAGE_PMD_NR);
  1552. mod_zone_page_state(page_zone(page),
  1553. NR_ISOLATED_ANON + page_lru,
  1554. -HPAGE_PMD_NR);
  1555. return isolated;
  1556. out_fail:
  1557. count_vm_events(PGMIGRATE_FAIL, HPAGE_PMD_NR);
  1558. out_dropref:
  1559. entry = pmd_mknonnuma(entry);
  1560. set_pmd_at(mm, haddr, pmd, entry);
  1561. update_mmu_cache_pmd(vma, address, &entry);
  1562. unlock_page(page);
  1563. put_page(page);
  1564. return 0;
  1565. }
  1566. #endif /* CONFIG_NUMA_BALANCING */
  1567. #endif /* CONFIG_NUMA */