rmap.c 51 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807
  1. /*
  2. * mm/rmap.c - physical to virtual reverse mappings
  3. *
  4. * Copyright 2001, Rik van Riel <riel@conectiva.com.br>
  5. * Released under the General Public License (GPL).
  6. *
  7. * Simple, low overhead reverse mapping scheme.
  8. * Please try to keep this thing as modular as possible.
  9. *
  10. * Provides methods for unmapping each kind of mapped page:
  11. * the anon methods track anonymous pages, and
  12. * the file methods track pages belonging to an inode.
  13. *
  14. * Original design by Rik van Riel <riel@conectiva.com.br> 2001
  15. * File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004
  16. * Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004
  17. * Contributions by Hugh Dickins 2003, 2004
  18. */
  19. /*
  20. * Lock ordering in mm:
  21. *
  22. * inode->i_mutex (while writing or truncating, not reading or faulting)
  23. * mm->mmap_sem
  24. * page->flags PG_locked (lock_page)
  25. * mapping->i_mmap_mutex
  26. * anon_vma->mutex
  27. * mm->page_table_lock or pte_lock
  28. * zone->lru_lock (in mark_page_accessed, isolate_lru_page)
  29. * swap_lock (in swap_duplicate, swap_info_get)
  30. * mmlist_lock (in mmput, drain_mmlist and others)
  31. * mapping->private_lock (in __set_page_dirty_buffers)
  32. * inode->i_lock (in set_page_dirty's __mark_inode_dirty)
  33. * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty)
  34. * sb_lock (within inode_lock in fs/fs-writeback.c)
  35. * mapping->tree_lock (widely used, in set_page_dirty,
  36. * in arch-dependent flush_dcache_mmap_lock,
  37. * within bdi.wb->list_lock in __sync_single_inode)
  38. *
  39. * anon_vma->mutex,mapping->i_mutex (memory_failure, collect_procs_anon)
  40. * ->tasklist_lock
  41. * pte map lock
  42. */
  43. #include <linux/mm.h>
  44. #include <linux/pagemap.h>
  45. #include <linux/swap.h>
  46. #include <linux/swapops.h>
  47. #include <linux/slab.h>
  48. #include <linux/init.h>
  49. #include <linux/ksm.h>
  50. #include <linux/rmap.h>
  51. #include <linux/rcupdate.h>
  52. #include <linux/export.h>
  53. #include <linux/memcontrol.h>
  54. #include <linux/mmu_notifier.h>
  55. #include <linux/migrate.h>
  56. #include <linux/hugetlb.h>
  57. #include <linux/backing-dev.h>
  58. #include <asm/tlbflush.h>
  59. #include "internal.h"
  60. static struct kmem_cache *anon_vma_cachep;
  61. static struct kmem_cache *anon_vma_chain_cachep;
  62. static inline struct anon_vma *anon_vma_alloc(void)
  63. {
  64. struct anon_vma *anon_vma;
  65. anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL);
  66. if (anon_vma) {
  67. atomic_set(&anon_vma->refcount, 1);
  68. /*
  69. * Initialise the anon_vma root to point to itself. If called
  70. * from fork, the root will be reset to the parents anon_vma.
  71. */
  72. anon_vma->root = anon_vma;
  73. }
  74. return anon_vma;
  75. }
  76. static inline void anon_vma_free(struct anon_vma *anon_vma)
  77. {
  78. VM_BUG_ON(atomic_read(&anon_vma->refcount));
  79. /*
  80. * Synchronize against page_lock_anon_vma() such that
  81. * we can safely hold the lock without the anon_vma getting
  82. * freed.
  83. *
  84. * Relies on the full mb implied by the atomic_dec_and_test() from
  85. * put_anon_vma() against the acquire barrier implied by
  86. * mutex_trylock() from page_lock_anon_vma(). This orders:
  87. *
  88. * page_lock_anon_vma() VS put_anon_vma()
  89. * mutex_trylock() atomic_dec_and_test()
  90. * LOCK MB
  91. * atomic_read() mutex_is_locked()
  92. *
  93. * LOCK should suffice since the actual taking of the lock must
  94. * happen _before_ what follows.
  95. */
  96. if (mutex_is_locked(&anon_vma->root->mutex)) {
  97. anon_vma_lock(anon_vma);
  98. anon_vma_unlock(anon_vma);
  99. }
  100. kmem_cache_free(anon_vma_cachep, anon_vma);
  101. }
  102. static inline struct anon_vma_chain *anon_vma_chain_alloc(gfp_t gfp)
  103. {
  104. return kmem_cache_alloc(anon_vma_chain_cachep, gfp);
  105. }
  106. static void anon_vma_chain_free(struct anon_vma_chain *anon_vma_chain)
  107. {
  108. kmem_cache_free(anon_vma_chain_cachep, anon_vma_chain);
  109. }
  110. static void anon_vma_chain_link(struct vm_area_struct *vma,
  111. struct anon_vma_chain *avc,
  112. struct anon_vma *anon_vma)
  113. {
  114. avc->vma = vma;
  115. avc->anon_vma = anon_vma;
  116. list_add(&avc->same_vma, &vma->anon_vma_chain);
  117. anon_vma_interval_tree_insert(avc, &anon_vma->rb_root);
  118. }
  119. /**
  120. * anon_vma_prepare - attach an anon_vma to a memory region
  121. * @vma: the memory region in question
  122. *
  123. * This makes sure the memory mapping described by 'vma' has
  124. * an 'anon_vma' attached to it, so that we can associate the
  125. * anonymous pages mapped into it with that anon_vma.
  126. *
  127. * The common case will be that we already have one, but if
  128. * not we either need to find an adjacent mapping that we
  129. * can re-use the anon_vma from (very common when the only
  130. * reason for splitting a vma has been mprotect()), or we
  131. * allocate a new one.
  132. *
  133. * Anon-vma allocations are very subtle, because we may have
  134. * optimistically looked up an anon_vma in page_lock_anon_vma()
  135. * and that may actually touch the spinlock even in the newly
  136. * allocated vma (it depends on RCU to make sure that the
  137. * anon_vma isn't actually destroyed).
  138. *
  139. * As a result, we need to do proper anon_vma locking even
  140. * for the new allocation. At the same time, we do not want
  141. * to do any locking for the common case of already having
  142. * an anon_vma.
  143. *
  144. * This must be called with the mmap_sem held for reading.
  145. */
  146. int anon_vma_prepare(struct vm_area_struct *vma)
  147. {
  148. struct anon_vma *anon_vma = vma->anon_vma;
  149. struct anon_vma_chain *avc;
  150. might_sleep();
  151. if (unlikely(!anon_vma)) {
  152. struct mm_struct *mm = vma->vm_mm;
  153. struct anon_vma *allocated;
  154. avc = anon_vma_chain_alloc(GFP_KERNEL);
  155. if (!avc)
  156. goto out_enomem;
  157. anon_vma = find_mergeable_anon_vma(vma);
  158. allocated = NULL;
  159. if (!anon_vma) {
  160. anon_vma = anon_vma_alloc();
  161. if (unlikely(!anon_vma))
  162. goto out_enomem_free_avc;
  163. allocated = anon_vma;
  164. }
  165. anon_vma_lock(anon_vma);
  166. /* page_table_lock to protect against threads */
  167. spin_lock(&mm->page_table_lock);
  168. if (likely(!vma->anon_vma)) {
  169. vma->anon_vma = anon_vma;
  170. anon_vma_chain_link(vma, avc, anon_vma);
  171. allocated = NULL;
  172. avc = NULL;
  173. }
  174. spin_unlock(&mm->page_table_lock);
  175. anon_vma_unlock(anon_vma);
  176. if (unlikely(allocated))
  177. put_anon_vma(allocated);
  178. if (unlikely(avc))
  179. anon_vma_chain_free(avc);
  180. }
  181. return 0;
  182. out_enomem_free_avc:
  183. anon_vma_chain_free(avc);
  184. out_enomem:
  185. return -ENOMEM;
  186. }
  187. /*
  188. * This is a useful helper function for locking the anon_vma root as
  189. * we traverse the vma->anon_vma_chain, looping over anon_vma's that
  190. * have the same vma.
  191. *
  192. * Such anon_vma's should have the same root, so you'd expect to see
  193. * just a single mutex_lock for the whole traversal.
  194. */
  195. static inline struct anon_vma *lock_anon_vma_root(struct anon_vma *root, struct anon_vma *anon_vma)
  196. {
  197. struct anon_vma *new_root = anon_vma->root;
  198. if (new_root != root) {
  199. if (WARN_ON_ONCE(root))
  200. mutex_unlock(&root->mutex);
  201. root = new_root;
  202. mutex_lock(&root->mutex);
  203. }
  204. return root;
  205. }
  206. static inline void unlock_anon_vma_root(struct anon_vma *root)
  207. {
  208. if (root)
  209. mutex_unlock(&root->mutex);
  210. }
  211. /*
  212. * Attach the anon_vmas from src to dst.
  213. * Returns 0 on success, -ENOMEM on failure.
  214. */
  215. int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src)
  216. {
  217. struct anon_vma_chain *avc, *pavc;
  218. struct anon_vma *root = NULL;
  219. list_for_each_entry_reverse(pavc, &src->anon_vma_chain, same_vma) {
  220. struct anon_vma *anon_vma;
  221. avc = anon_vma_chain_alloc(GFP_NOWAIT | __GFP_NOWARN);
  222. if (unlikely(!avc)) {
  223. unlock_anon_vma_root(root);
  224. root = NULL;
  225. avc = anon_vma_chain_alloc(GFP_KERNEL);
  226. if (!avc)
  227. goto enomem_failure;
  228. }
  229. anon_vma = pavc->anon_vma;
  230. root = lock_anon_vma_root(root, anon_vma);
  231. anon_vma_chain_link(dst, avc, anon_vma);
  232. }
  233. unlock_anon_vma_root(root);
  234. return 0;
  235. enomem_failure:
  236. unlink_anon_vmas(dst);
  237. return -ENOMEM;
  238. }
  239. /*
  240. * Attach vma to its own anon_vma, as well as to the anon_vmas that
  241. * the corresponding VMA in the parent process is attached to.
  242. * Returns 0 on success, non-zero on failure.
  243. */
  244. int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
  245. {
  246. struct anon_vma_chain *avc;
  247. struct anon_vma *anon_vma;
  248. /* Don't bother if the parent process has no anon_vma here. */
  249. if (!pvma->anon_vma)
  250. return 0;
  251. /*
  252. * First, attach the new VMA to the parent VMA's anon_vmas,
  253. * so rmap can find non-COWed pages in child processes.
  254. */
  255. if (anon_vma_clone(vma, pvma))
  256. return -ENOMEM;
  257. /* Then add our own anon_vma. */
  258. anon_vma = anon_vma_alloc();
  259. if (!anon_vma)
  260. goto out_error;
  261. avc = anon_vma_chain_alloc(GFP_KERNEL);
  262. if (!avc)
  263. goto out_error_free_anon_vma;
  264. /*
  265. * The root anon_vma's spinlock is the lock actually used when we
  266. * lock any of the anon_vmas in this anon_vma tree.
  267. */
  268. anon_vma->root = pvma->anon_vma->root;
  269. /*
  270. * With refcounts, an anon_vma can stay around longer than the
  271. * process it belongs to. The root anon_vma needs to be pinned until
  272. * this anon_vma is freed, because the lock lives in the root.
  273. */
  274. get_anon_vma(anon_vma->root);
  275. /* Mark this anon_vma as the one where our new (COWed) pages go. */
  276. vma->anon_vma = anon_vma;
  277. anon_vma_lock(anon_vma);
  278. anon_vma_chain_link(vma, avc, anon_vma);
  279. anon_vma_unlock(anon_vma);
  280. return 0;
  281. out_error_free_anon_vma:
  282. put_anon_vma(anon_vma);
  283. out_error:
  284. unlink_anon_vmas(vma);
  285. return -ENOMEM;
  286. }
  287. void unlink_anon_vmas(struct vm_area_struct *vma)
  288. {
  289. struct anon_vma_chain *avc, *next;
  290. struct anon_vma *root = NULL;
  291. /*
  292. * Unlink each anon_vma chained to the VMA. This list is ordered
  293. * from newest to oldest, ensuring the root anon_vma gets freed last.
  294. */
  295. list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
  296. struct anon_vma *anon_vma = avc->anon_vma;
  297. root = lock_anon_vma_root(root, anon_vma);
  298. anon_vma_interval_tree_remove(avc, &anon_vma->rb_root);
  299. /*
  300. * Leave empty anon_vmas on the list - we'll need
  301. * to free them outside the lock.
  302. */
  303. if (RB_EMPTY_ROOT(&anon_vma->rb_root))
  304. continue;
  305. list_del(&avc->same_vma);
  306. anon_vma_chain_free(avc);
  307. }
  308. unlock_anon_vma_root(root);
  309. /*
  310. * Iterate the list once more, it now only contains empty and unlinked
  311. * anon_vmas, destroy them. Could not do before due to __put_anon_vma()
  312. * needing to acquire the anon_vma->root->mutex.
  313. */
  314. list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
  315. struct anon_vma *anon_vma = avc->anon_vma;
  316. put_anon_vma(anon_vma);
  317. list_del(&avc->same_vma);
  318. anon_vma_chain_free(avc);
  319. }
  320. }
  321. static void anon_vma_ctor(void *data)
  322. {
  323. struct anon_vma *anon_vma = data;
  324. mutex_init(&anon_vma->mutex);
  325. atomic_set(&anon_vma->refcount, 0);
  326. anon_vma->rb_root = RB_ROOT;
  327. }
  328. void __init anon_vma_init(void)
  329. {
  330. anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma),
  331. 0, SLAB_DESTROY_BY_RCU|SLAB_PANIC, anon_vma_ctor);
  332. anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain, SLAB_PANIC);
  333. }
  334. /*
  335. * Getting a lock on a stable anon_vma from a page off the LRU is tricky!
  336. *
  337. * Since there is no serialization what so ever against page_remove_rmap()
  338. * the best this function can do is return a locked anon_vma that might
  339. * have been relevant to this page.
  340. *
  341. * The page might have been remapped to a different anon_vma or the anon_vma
  342. * returned may already be freed (and even reused).
  343. *
  344. * In case it was remapped to a different anon_vma, the new anon_vma will be a
  345. * child of the old anon_vma, and the anon_vma lifetime rules will therefore
  346. * ensure that any anon_vma obtained from the page will still be valid for as
  347. * long as we observe page_mapped() [ hence all those page_mapped() tests ].
  348. *
  349. * All users of this function must be very careful when walking the anon_vma
  350. * chain and verify that the page in question is indeed mapped in it
  351. * [ something equivalent to page_mapped_in_vma() ].
  352. *
  353. * Since anon_vma's slab is DESTROY_BY_RCU and we know from page_remove_rmap()
  354. * that the anon_vma pointer from page->mapping is valid if there is a
  355. * mapcount, we can dereference the anon_vma after observing those.
  356. */
  357. struct anon_vma *page_get_anon_vma(struct page *page)
  358. {
  359. struct anon_vma *anon_vma = NULL;
  360. unsigned long anon_mapping;
  361. rcu_read_lock();
  362. anon_mapping = (unsigned long) ACCESS_ONCE(page->mapping);
  363. if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
  364. goto out;
  365. if (!page_mapped(page))
  366. goto out;
  367. anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
  368. if (!atomic_inc_not_zero(&anon_vma->refcount)) {
  369. anon_vma = NULL;
  370. goto out;
  371. }
  372. /*
  373. * If this page is still mapped, then its anon_vma cannot have been
  374. * freed. But if it has been unmapped, we have no security against the
  375. * anon_vma structure being freed and reused (for another anon_vma:
  376. * SLAB_DESTROY_BY_RCU guarantees that - so the atomic_inc_not_zero()
  377. * above cannot corrupt).
  378. */
  379. if (!page_mapped(page)) {
  380. put_anon_vma(anon_vma);
  381. anon_vma = NULL;
  382. }
  383. out:
  384. rcu_read_unlock();
  385. return anon_vma;
  386. }
  387. /*
  388. * Similar to page_get_anon_vma() except it locks the anon_vma.
  389. *
  390. * Its a little more complex as it tries to keep the fast path to a single
  391. * atomic op -- the trylock. If we fail the trylock, we fall back to getting a
  392. * reference like with page_get_anon_vma() and then block on the mutex.
  393. */
  394. struct anon_vma *page_lock_anon_vma(struct page *page)
  395. {
  396. struct anon_vma *anon_vma = NULL;
  397. struct anon_vma *root_anon_vma;
  398. unsigned long anon_mapping;
  399. rcu_read_lock();
  400. anon_mapping = (unsigned long) ACCESS_ONCE(page->mapping);
  401. if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
  402. goto out;
  403. if (!page_mapped(page))
  404. goto out;
  405. anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
  406. root_anon_vma = ACCESS_ONCE(anon_vma->root);
  407. if (mutex_trylock(&root_anon_vma->mutex)) {
  408. /*
  409. * If the page is still mapped, then this anon_vma is still
  410. * its anon_vma, and holding the mutex ensures that it will
  411. * not go away, see anon_vma_free().
  412. */
  413. if (!page_mapped(page)) {
  414. mutex_unlock(&root_anon_vma->mutex);
  415. anon_vma = NULL;
  416. }
  417. goto out;
  418. }
  419. /* trylock failed, we got to sleep */
  420. if (!atomic_inc_not_zero(&anon_vma->refcount)) {
  421. anon_vma = NULL;
  422. goto out;
  423. }
  424. if (!page_mapped(page)) {
  425. put_anon_vma(anon_vma);
  426. anon_vma = NULL;
  427. goto out;
  428. }
  429. /* we pinned the anon_vma, its safe to sleep */
  430. rcu_read_unlock();
  431. anon_vma_lock(anon_vma);
  432. if (atomic_dec_and_test(&anon_vma->refcount)) {
  433. /*
  434. * Oops, we held the last refcount, release the lock
  435. * and bail -- can't simply use put_anon_vma() because
  436. * we'll deadlock on the anon_vma_lock() recursion.
  437. */
  438. anon_vma_unlock(anon_vma);
  439. __put_anon_vma(anon_vma);
  440. anon_vma = NULL;
  441. }
  442. return anon_vma;
  443. out:
  444. rcu_read_unlock();
  445. return anon_vma;
  446. }
  447. void page_unlock_anon_vma(struct anon_vma *anon_vma)
  448. {
  449. anon_vma_unlock(anon_vma);
  450. }
  451. /*
  452. * At what user virtual address is page expected in @vma?
  453. */
  454. static inline unsigned long
  455. __vma_address(struct page *page, struct vm_area_struct *vma)
  456. {
  457. pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
  458. if (unlikely(is_vm_hugetlb_page(vma)))
  459. pgoff = page->index << huge_page_order(page_hstate(page));
  460. return vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
  461. }
  462. inline unsigned long
  463. vma_address(struct page *page, struct vm_area_struct *vma)
  464. {
  465. unsigned long address = __vma_address(page, vma);
  466. /* page should be within @vma mapping range */
  467. VM_BUG_ON(address < vma->vm_start || address >= vma->vm_end);
  468. return address;
  469. }
  470. /*
  471. * At what user virtual address is page expected in vma?
  472. * Caller should check the page is actually part of the vma.
  473. */
  474. unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma)
  475. {
  476. unsigned long address;
  477. if (PageAnon(page)) {
  478. struct anon_vma *page__anon_vma = page_anon_vma(page);
  479. /*
  480. * Note: swapoff's unuse_vma() is more efficient with this
  481. * check, and needs it to match anon_vma when KSM is active.
  482. */
  483. if (!vma->anon_vma || !page__anon_vma ||
  484. vma->anon_vma->root != page__anon_vma->root)
  485. return -EFAULT;
  486. } else if (page->mapping && !(vma->vm_flags & VM_NONLINEAR)) {
  487. if (!vma->vm_file ||
  488. vma->vm_file->f_mapping != page->mapping)
  489. return -EFAULT;
  490. } else
  491. return -EFAULT;
  492. address = __vma_address(page, vma);
  493. if (unlikely(address < vma->vm_start || address >= vma->vm_end))
  494. return -EFAULT;
  495. return address;
  496. }
  497. pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address)
  498. {
  499. pgd_t *pgd;
  500. pud_t *pud;
  501. pmd_t *pmd = NULL;
  502. pgd = pgd_offset(mm, address);
  503. if (!pgd_present(*pgd))
  504. goto out;
  505. pud = pud_offset(pgd, address);
  506. if (!pud_present(*pud))
  507. goto out;
  508. pmd = pmd_offset(pud, address);
  509. if (!pmd_present(*pmd))
  510. pmd = NULL;
  511. out:
  512. return pmd;
  513. }
  514. /*
  515. * Check that @page is mapped at @address into @mm.
  516. *
  517. * If @sync is false, page_check_address may perform a racy check to avoid
  518. * the page table lock when the pte is not present (helpful when reclaiming
  519. * highly shared pages).
  520. *
  521. * On success returns with pte mapped and locked.
  522. */
  523. pte_t *__page_check_address(struct page *page, struct mm_struct *mm,
  524. unsigned long address, spinlock_t **ptlp, int sync)
  525. {
  526. pmd_t *pmd;
  527. pte_t *pte;
  528. spinlock_t *ptl;
  529. if (unlikely(PageHuge(page))) {
  530. pte = huge_pte_offset(mm, address);
  531. ptl = &mm->page_table_lock;
  532. goto check;
  533. }
  534. pmd = mm_find_pmd(mm, address);
  535. if (!pmd)
  536. return NULL;
  537. if (pmd_trans_huge(*pmd))
  538. return NULL;
  539. pte = pte_offset_map(pmd, address);
  540. /* Make a quick check before getting the lock */
  541. if (!sync && !pte_present(*pte)) {
  542. pte_unmap(pte);
  543. return NULL;
  544. }
  545. ptl = pte_lockptr(mm, pmd);
  546. check:
  547. spin_lock(ptl);
  548. if (pte_present(*pte) && page_to_pfn(page) == pte_pfn(*pte)) {
  549. *ptlp = ptl;
  550. return pte;
  551. }
  552. pte_unmap_unlock(pte, ptl);
  553. return NULL;
  554. }
  555. /**
  556. * page_mapped_in_vma - check whether a page is really mapped in a VMA
  557. * @page: the page to test
  558. * @vma: the VMA to test
  559. *
  560. * Returns 1 if the page is mapped into the page tables of the VMA, 0
  561. * if the page is not mapped into the page tables of this VMA. Only
  562. * valid for normal file or anonymous VMAs.
  563. */
  564. int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma)
  565. {
  566. unsigned long address;
  567. pte_t *pte;
  568. spinlock_t *ptl;
  569. address = __vma_address(page, vma);
  570. if (unlikely(address < vma->vm_start || address >= vma->vm_end))
  571. return 0;
  572. pte = page_check_address(page, vma->vm_mm, address, &ptl, 1);
  573. if (!pte) /* the page is not in this mm */
  574. return 0;
  575. pte_unmap_unlock(pte, ptl);
  576. return 1;
  577. }
  578. /*
  579. * Subfunctions of page_referenced: page_referenced_one called
  580. * repeatedly from either page_referenced_anon or page_referenced_file.
  581. */
  582. int page_referenced_one(struct page *page, struct vm_area_struct *vma,
  583. unsigned long address, unsigned int *mapcount,
  584. unsigned long *vm_flags)
  585. {
  586. struct mm_struct *mm = vma->vm_mm;
  587. int referenced = 0;
  588. if (unlikely(PageTransHuge(page))) {
  589. pmd_t *pmd;
  590. spin_lock(&mm->page_table_lock);
  591. /*
  592. * rmap might return false positives; we must filter
  593. * these out using page_check_address_pmd().
  594. */
  595. pmd = page_check_address_pmd(page, mm, address,
  596. PAGE_CHECK_ADDRESS_PMD_FLAG);
  597. if (!pmd) {
  598. spin_unlock(&mm->page_table_lock);
  599. goto out;
  600. }
  601. if (vma->vm_flags & VM_LOCKED) {
  602. spin_unlock(&mm->page_table_lock);
  603. *mapcount = 0; /* break early from loop */
  604. *vm_flags |= VM_LOCKED;
  605. goto out;
  606. }
  607. /* go ahead even if the pmd is pmd_trans_splitting() */
  608. if (pmdp_clear_flush_young_notify(vma, address, pmd))
  609. referenced++;
  610. spin_unlock(&mm->page_table_lock);
  611. } else {
  612. pte_t *pte;
  613. spinlock_t *ptl;
  614. /*
  615. * rmap might return false positives; we must filter
  616. * these out using page_check_address().
  617. */
  618. pte = page_check_address(page, mm, address, &ptl, 0);
  619. if (!pte)
  620. goto out;
  621. if (vma->vm_flags & VM_LOCKED) {
  622. pte_unmap_unlock(pte, ptl);
  623. *mapcount = 0; /* break early from loop */
  624. *vm_flags |= VM_LOCKED;
  625. goto out;
  626. }
  627. if (ptep_clear_flush_young_notify(vma, address, pte)) {
  628. /*
  629. * Don't treat a reference through a sequentially read
  630. * mapping as such. If the page has been used in
  631. * another mapping, we will catch it; if this other
  632. * mapping is already gone, the unmap path will have
  633. * set PG_referenced or activated the page.
  634. */
  635. if (likely(!VM_SequentialReadHint(vma)))
  636. referenced++;
  637. }
  638. pte_unmap_unlock(pte, ptl);
  639. }
  640. (*mapcount)--;
  641. if (referenced)
  642. *vm_flags |= vma->vm_flags;
  643. out:
  644. return referenced;
  645. }
  646. static int page_referenced_anon(struct page *page,
  647. struct mem_cgroup *memcg,
  648. unsigned long *vm_flags)
  649. {
  650. unsigned int mapcount;
  651. struct anon_vma *anon_vma;
  652. pgoff_t pgoff;
  653. struct anon_vma_chain *avc;
  654. int referenced = 0;
  655. anon_vma = page_lock_anon_vma(page);
  656. if (!anon_vma)
  657. return referenced;
  658. mapcount = page_mapcount(page);
  659. pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
  660. anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff, pgoff) {
  661. struct vm_area_struct *vma = avc->vma;
  662. unsigned long address = vma_address(page, vma);
  663. /*
  664. * If we are reclaiming on behalf of a cgroup, skip
  665. * counting on behalf of references from different
  666. * cgroups
  667. */
  668. if (memcg && !mm_match_cgroup(vma->vm_mm, memcg))
  669. continue;
  670. referenced += page_referenced_one(page, vma, address,
  671. &mapcount, vm_flags);
  672. if (!mapcount)
  673. break;
  674. }
  675. page_unlock_anon_vma(anon_vma);
  676. return referenced;
  677. }
  678. /**
  679. * page_referenced_file - referenced check for object-based rmap
  680. * @page: the page we're checking references on.
  681. * @memcg: target memory control group
  682. * @vm_flags: collect encountered vma->vm_flags who actually referenced the page
  683. *
  684. * For an object-based mapped page, find all the places it is mapped and
  685. * check/clear the referenced flag. This is done by following the page->mapping
  686. * pointer, then walking the chain of vmas it holds. It returns the number
  687. * of references it found.
  688. *
  689. * This function is only called from page_referenced for object-based pages.
  690. */
  691. static int page_referenced_file(struct page *page,
  692. struct mem_cgroup *memcg,
  693. unsigned long *vm_flags)
  694. {
  695. unsigned int mapcount;
  696. struct address_space *mapping = page->mapping;
  697. pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
  698. struct vm_area_struct *vma;
  699. int referenced = 0;
  700. /*
  701. * The caller's checks on page->mapping and !PageAnon have made
  702. * sure that this is a file page: the check for page->mapping
  703. * excludes the case just before it gets set on an anon page.
  704. */
  705. BUG_ON(PageAnon(page));
  706. /*
  707. * The page lock not only makes sure that page->mapping cannot
  708. * suddenly be NULLified by truncation, it makes sure that the
  709. * structure at mapping cannot be freed and reused yet,
  710. * so we can safely take mapping->i_mmap_mutex.
  711. */
  712. BUG_ON(!PageLocked(page));
  713. mutex_lock(&mapping->i_mmap_mutex);
  714. /*
  715. * i_mmap_mutex does not stabilize mapcount at all, but mapcount
  716. * is more likely to be accurate if we note it after spinning.
  717. */
  718. mapcount = page_mapcount(page);
  719. vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
  720. unsigned long address = vma_address(page, vma);
  721. /*
  722. * If we are reclaiming on behalf of a cgroup, skip
  723. * counting on behalf of references from different
  724. * cgroups
  725. */
  726. if (memcg && !mm_match_cgroup(vma->vm_mm, memcg))
  727. continue;
  728. referenced += page_referenced_one(page, vma, address,
  729. &mapcount, vm_flags);
  730. if (!mapcount)
  731. break;
  732. }
  733. mutex_unlock(&mapping->i_mmap_mutex);
  734. return referenced;
  735. }
  736. /**
  737. * page_referenced - test if the page was referenced
  738. * @page: the page to test
  739. * @is_locked: caller holds lock on the page
  740. * @memcg: target memory cgroup
  741. * @vm_flags: collect encountered vma->vm_flags who actually referenced the page
  742. *
  743. * Quick test_and_clear_referenced for all mappings to a page,
  744. * returns the number of ptes which referenced the page.
  745. */
  746. int page_referenced(struct page *page,
  747. int is_locked,
  748. struct mem_cgroup *memcg,
  749. unsigned long *vm_flags)
  750. {
  751. int referenced = 0;
  752. int we_locked = 0;
  753. *vm_flags = 0;
  754. if (page_mapped(page) && page_rmapping(page)) {
  755. if (!is_locked && (!PageAnon(page) || PageKsm(page))) {
  756. we_locked = trylock_page(page);
  757. if (!we_locked) {
  758. referenced++;
  759. goto out;
  760. }
  761. }
  762. if (unlikely(PageKsm(page)))
  763. referenced += page_referenced_ksm(page, memcg,
  764. vm_flags);
  765. else if (PageAnon(page))
  766. referenced += page_referenced_anon(page, memcg,
  767. vm_flags);
  768. else if (page->mapping)
  769. referenced += page_referenced_file(page, memcg,
  770. vm_flags);
  771. if (we_locked)
  772. unlock_page(page);
  773. if (page_test_and_clear_young(page_to_pfn(page)))
  774. referenced++;
  775. }
  776. out:
  777. return referenced;
  778. }
  779. static int page_mkclean_one(struct page *page, struct vm_area_struct *vma,
  780. unsigned long address)
  781. {
  782. struct mm_struct *mm = vma->vm_mm;
  783. pte_t *pte;
  784. spinlock_t *ptl;
  785. int ret = 0;
  786. pte = page_check_address(page, mm, address, &ptl, 1);
  787. if (!pte)
  788. goto out;
  789. if (pte_dirty(*pte) || pte_write(*pte)) {
  790. pte_t entry;
  791. flush_cache_page(vma, address, pte_pfn(*pte));
  792. entry = ptep_clear_flush(vma, address, pte);
  793. entry = pte_wrprotect(entry);
  794. entry = pte_mkclean(entry);
  795. set_pte_at(mm, address, pte, entry);
  796. ret = 1;
  797. }
  798. pte_unmap_unlock(pte, ptl);
  799. if (ret)
  800. mmu_notifier_invalidate_page(mm, address);
  801. out:
  802. return ret;
  803. }
  804. static int page_mkclean_file(struct address_space *mapping, struct page *page)
  805. {
  806. pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
  807. struct vm_area_struct *vma;
  808. int ret = 0;
  809. BUG_ON(PageAnon(page));
  810. mutex_lock(&mapping->i_mmap_mutex);
  811. vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
  812. if (vma->vm_flags & VM_SHARED) {
  813. unsigned long address = vma_address(page, vma);
  814. ret += page_mkclean_one(page, vma, address);
  815. }
  816. }
  817. mutex_unlock(&mapping->i_mmap_mutex);
  818. return ret;
  819. }
  820. int page_mkclean(struct page *page)
  821. {
  822. int ret = 0;
  823. BUG_ON(!PageLocked(page));
  824. if (page_mapped(page)) {
  825. struct address_space *mapping = page_mapping(page);
  826. if (mapping)
  827. ret = page_mkclean_file(mapping, page);
  828. }
  829. return ret;
  830. }
  831. EXPORT_SYMBOL_GPL(page_mkclean);
  832. /**
  833. * page_move_anon_rmap - move a page to our anon_vma
  834. * @page: the page to move to our anon_vma
  835. * @vma: the vma the page belongs to
  836. * @address: the user virtual address mapped
  837. *
  838. * When a page belongs exclusively to one process after a COW event,
  839. * that page can be moved into the anon_vma that belongs to just that
  840. * process, so the rmap code will not search the parent or sibling
  841. * processes.
  842. */
  843. void page_move_anon_rmap(struct page *page,
  844. struct vm_area_struct *vma, unsigned long address)
  845. {
  846. struct anon_vma *anon_vma = vma->anon_vma;
  847. VM_BUG_ON(!PageLocked(page));
  848. VM_BUG_ON(!anon_vma);
  849. VM_BUG_ON(page->index != linear_page_index(vma, address));
  850. anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
  851. page->mapping = (struct address_space *) anon_vma;
  852. }
  853. /**
  854. * __page_set_anon_rmap - set up new anonymous rmap
  855. * @page: Page to add to rmap
  856. * @vma: VM area to add page to.
  857. * @address: User virtual address of the mapping
  858. * @exclusive: the page is exclusively owned by the current process
  859. */
  860. static void __page_set_anon_rmap(struct page *page,
  861. struct vm_area_struct *vma, unsigned long address, int exclusive)
  862. {
  863. struct anon_vma *anon_vma = vma->anon_vma;
  864. BUG_ON(!anon_vma);
  865. if (PageAnon(page))
  866. return;
  867. /*
  868. * If the page isn't exclusively mapped into this vma,
  869. * we must use the _oldest_ possible anon_vma for the
  870. * page mapping!
  871. */
  872. if (!exclusive)
  873. anon_vma = anon_vma->root;
  874. anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
  875. page->mapping = (struct address_space *) anon_vma;
  876. page->index = linear_page_index(vma, address);
  877. }
  878. /**
  879. * __page_check_anon_rmap - sanity check anonymous rmap addition
  880. * @page: the page to add the mapping to
  881. * @vma: the vm area in which the mapping is added
  882. * @address: the user virtual address mapped
  883. */
  884. static void __page_check_anon_rmap(struct page *page,
  885. struct vm_area_struct *vma, unsigned long address)
  886. {
  887. #ifdef CONFIG_DEBUG_VM
  888. /*
  889. * The page's anon-rmap details (mapping and index) are guaranteed to
  890. * be set up correctly at this point.
  891. *
  892. * We have exclusion against page_add_anon_rmap because the caller
  893. * always holds the page locked, except if called from page_dup_rmap,
  894. * in which case the page is already known to be setup.
  895. *
  896. * We have exclusion against page_add_new_anon_rmap because those pages
  897. * are initially only visible via the pagetables, and the pte is locked
  898. * over the call to page_add_new_anon_rmap.
  899. */
  900. BUG_ON(page_anon_vma(page)->root != vma->anon_vma->root);
  901. BUG_ON(page->index != linear_page_index(vma, address));
  902. #endif
  903. }
  904. /**
  905. * page_add_anon_rmap - add pte mapping to an anonymous page
  906. * @page: the page to add the mapping to
  907. * @vma: the vm area in which the mapping is added
  908. * @address: the user virtual address mapped
  909. *
  910. * The caller needs to hold the pte lock, and the page must be locked in
  911. * the anon_vma case: to serialize mapping,index checking after setting,
  912. * and to ensure that PageAnon is not being upgraded racily to PageKsm
  913. * (but PageKsm is never downgraded to PageAnon).
  914. */
  915. void page_add_anon_rmap(struct page *page,
  916. struct vm_area_struct *vma, unsigned long address)
  917. {
  918. do_page_add_anon_rmap(page, vma, address, 0);
  919. }
  920. /*
  921. * Special version of the above for do_swap_page, which often runs
  922. * into pages that are exclusively owned by the current process.
  923. * Everybody else should continue to use page_add_anon_rmap above.
  924. */
  925. void do_page_add_anon_rmap(struct page *page,
  926. struct vm_area_struct *vma, unsigned long address, int exclusive)
  927. {
  928. int first = atomic_inc_and_test(&page->_mapcount);
  929. if (first) {
  930. if (!PageTransHuge(page))
  931. __inc_zone_page_state(page, NR_ANON_PAGES);
  932. else
  933. __inc_zone_page_state(page,
  934. NR_ANON_TRANSPARENT_HUGEPAGES);
  935. }
  936. if (unlikely(PageKsm(page)))
  937. return;
  938. VM_BUG_ON(!PageLocked(page));
  939. /* address might be in next vma when migration races vma_adjust */
  940. if (first)
  941. __page_set_anon_rmap(page, vma, address, exclusive);
  942. else
  943. __page_check_anon_rmap(page, vma, address);
  944. }
  945. /**
  946. * page_add_new_anon_rmap - add pte mapping to a new anonymous page
  947. * @page: the page to add the mapping to
  948. * @vma: the vm area in which the mapping is added
  949. * @address: the user virtual address mapped
  950. *
  951. * Same as page_add_anon_rmap but must only be called on *new* pages.
  952. * This means the inc-and-test can be bypassed.
  953. * Page does not have to be locked.
  954. */
  955. void page_add_new_anon_rmap(struct page *page,
  956. struct vm_area_struct *vma, unsigned long address)
  957. {
  958. VM_BUG_ON(address < vma->vm_start || address >= vma->vm_end);
  959. SetPageSwapBacked(page);
  960. atomic_set(&page->_mapcount, 0); /* increment count (starts at -1) */
  961. if (!PageTransHuge(page))
  962. __inc_zone_page_state(page, NR_ANON_PAGES);
  963. else
  964. __inc_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES);
  965. __page_set_anon_rmap(page, vma, address, 1);
  966. if (!mlocked_vma_newpage(vma, page))
  967. lru_cache_add_lru(page, LRU_ACTIVE_ANON);
  968. else
  969. add_page_to_unevictable_list(page);
  970. }
  971. /**
  972. * page_add_file_rmap - add pte mapping to a file page
  973. * @page: the page to add the mapping to
  974. *
  975. * The caller needs to hold the pte lock.
  976. */
  977. void page_add_file_rmap(struct page *page)
  978. {
  979. bool locked;
  980. unsigned long flags;
  981. mem_cgroup_begin_update_page_stat(page, &locked, &flags);
  982. if (atomic_inc_and_test(&page->_mapcount)) {
  983. __inc_zone_page_state(page, NR_FILE_MAPPED);
  984. mem_cgroup_inc_page_stat(page, MEMCG_NR_FILE_MAPPED);
  985. }
  986. mem_cgroup_end_update_page_stat(page, &locked, &flags);
  987. }
  988. /**
  989. * page_remove_rmap - take down pte mapping from a page
  990. * @page: page to remove mapping from
  991. *
  992. * The caller needs to hold the pte lock.
  993. */
  994. void page_remove_rmap(struct page *page)
  995. {
  996. struct address_space *mapping = page_mapping(page);
  997. bool anon = PageAnon(page);
  998. bool locked;
  999. unsigned long flags;
  1000. /*
  1001. * The anon case has no mem_cgroup page_stat to update; but may
  1002. * uncharge_page() below, where the lock ordering can deadlock if
  1003. * we hold the lock against page_stat move: so avoid it on anon.
  1004. */
  1005. if (!anon)
  1006. mem_cgroup_begin_update_page_stat(page, &locked, &flags);
  1007. /* page still mapped by someone else? */
  1008. if (!atomic_add_negative(-1, &page->_mapcount))
  1009. goto out;
  1010. /*
  1011. * Now that the last pte has gone, s390 must transfer dirty
  1012. * flag from storage key to struct page. We can usually skip
  1013. * this if the page is anon, so about to be freed; but perhaps
  1014. * not if it's in swapcache - there might be another pte slot
  1015. * containing the swap entry, but page not yet written to swap.
  1016. *
  1017. * And we can skip it on file pages, so long as the filesystem
  1018. * participates in dirty tracking (note that this is not only an
  1019. * optimization but also solves problems caused by dirty flag in
  1020. * storage key getting set by a write from inside kernel); but need to
  1021. * catch shm and tmpfs and ramfs pages which have been modified since
  1022. * creation by read fault.
  1023. *
  1024. * Note that mapping must be decided above, before decrementing
  1025. * mapcount (which luckily provides a barrier): once page is unmapped,
  1026. * it could be truncated and page->mapping reset to NULL at any moment.
  1027. * Note also that we are relying on page_mapping(page) to set mapping
  1028. * to &swapper_space when PageSwapCache(page).
  1029. */
  1030. if (mapping && !mapping_cap_account_dirty(mapping) &&
  1031. page_test_and_clear_dirty(page_to_pfn(page), 1))
  1032. set_page_dirty(page);
  1033. /*
  1034. * Hugepages are not counted in NR_ANON_PAGES nor NR_FILE_MAPPED
  1035. * and not charged by memcg for now.
  1036. */
  1037. if (unlikely(PageHuge(page)))
  1038. goto out;
  1039. if (anon) {
  1040. mem_cgroup_uncharge_page(page);
  1041. if (!PageTransHuge(page))
  1042. __dec_zone_page_state(page, NR_ANON_PAGES);
  1043. else
  1044. __dec_zone_page_state(page,
  1045. NR_ANON_TRANSPARENT_HUGEPAGES);
  1046. } else {
  1047. __dec_zone_page_state(page, NR_FILE_MAPPED);
  1048. mem_cgroup_dec_page_stat(page, MEMCG_NR_FILE_MAPPED);
  1049. mem_cgroup_end_update_page_stat(page, &locked, &flags);
  1050. }
  1051. if (unlikely(PageMlocked(page)))
  1052. clear_page_mlock(page);
  1053. /*
  1054. * It would be tidy to reset the PageAnon mapping here,
  1055. * but that might overwrite a racing page_add_anon_rmap
  1056. * which increments mapcount after us but sets mapping
  1057. * before us: so leave the reset to free_hot_cold_page,
  1058. * and remember that it's only reliable while mapped.
  1059. * Leaving it set also helps swapoff to reinstate ptes
  1060. * faster for those pages still in swapcache.
  1061. */
  1062. return;
  1063. out:
  1064. if (!anon)
  1065. mem_cgroup_end_update_page_stat(page, &locked, &flags);
  1066. }
  1067. /*
  1068. * Subfunctions of try_to_unmap: try_to_unmap_one called
  1069. * repeatedly from try_to_unmap_ksm, try_to_unmap_anon or try_to_unmap_file.
  1070. */
  1071. int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
  1072. unsigned long address, enum ttu_flags flags)
  1073. {
  1074. struct mm_struct *mm = vma->vm_mm;
  1075. pte_t *pte;
  1076. pte_t pteval;
  1077. spinlock_t *ptl;
  1078. int ret = SWAP_AGAIN;
  1079. pte = page_check_address(page, mm, address, &ptl, 0);
  1080. if (!pte)
  1081. goto out;
  1082. /*
  1083. * If the page is mlock()d, we cannot swap it out.
  1084. * If it's recently referenced (perhaps page_referenced
  1085. * skipped over this mm) then we should reactivate it.
  1086. */
  1087. if (!(flags & TTU_IGNORE_MLOCK)) {
  1088. if (vma->vm_flags & VM_LOCKED)
  1089. goto out_mlock;
  1090. if (TTU_ACTION(flags) == TTU_MUNLOCK)
  1091. goto out_unmap;
  1092. }
  1093. if (!(flags & TTU_IGNORE_ACCESS)) {
  1094. if (ptep_clear_flush_young_notify(vma, address, pte)) {
  1095. ret = SWAP_FAIL;
  1096. goto out_unmap;
  1097. }
  1098. }
  1099. /* Nuke the page table entry. */
  1100. flush_cache_page(vma, address, page_to_pfn(page));
  1101. pteval = ptep_clear_flush(vma, address, pte);
  1102. /* Move the dirty bit to the physical page now the pte is gone. */
  1103. if (pte_dirty(pteval))
  1104. set_page_dirty(page);
  1105. /* Update high watermark before we lower rss */
  1106. update_hiwater_rss(mm);
  1107. if (PageHWPoison(page) && !(flags & TTU_IGNORE_HWPOISON)) {
  1108. if (PageAnon(page))
  1109. dec_mm_counter(mm, MM_ANONPAGES);
  1110. else
  1111. dec_mm_counter(mm, MM_FILEPAGES);
  1112. set_pte_at(mm, address, pte,
  1113. swp_entry_to_pte(make_hwpoison_entry(page)));
  1114. } else if (PageAnon(page)) {
  1115. swp_entry_t entry = { .val = page_private(page) };
  1116. if (PageSwapCache(page)) {
  1117. /*
  1118. * Store the swap location in the pte.
  1119. * See handle_pte_fault() ...
  1120. */
  1121. if (swap_duplicate(entry) < 0) {
  1122. set_pte_at(mm, address, pte, pteval);
  1123. ret = SWAP_FAIL;
  1124. goto out_unmap;
  1125. }
  1126. if (list_empty(&mm->mmlist)) {
  1127. spin_lock(&mmlist_lock);
  1128. if (list_empty(&mm->mmlist))
  1129. list_add(&mm->mmlist, &init_mm.mmlist);
  1130. spin_unlock(&mmlist_lock);
  1131. }
  1132. dec_mm_counter(mm, MM_ANONPAGES);
  1133. inc_mm_counter(mm, MM_SWAPENTS);
  1134. } else if (IS_ENABLED(CONFIG_MIGRATION)) {
  1135. /*
  1136. * Store the pfn of the page in a special migration
  1137. * pte. do_swap_page() will wait until the migration
  1138. * pte is removed and then restart fault handling.
  1139. */
  1140. BUG_ON(TTU_ACTION(flags) != TTU_MIGRATION);
  1141. entry = make_migration_entry(page, pte_write(pteval));
  1142. }
  1143. set_pte_at(mm, address, pte, swp_entry_to_pte(entry));
  1144. BUG_ON(pte_file(*pte));
  1145. } else if (IS_ENABLED(CONFIG_MIGRATION) &&
  1146. (TTU_ACTION(flags) == TTU_MIGRATION)) {
  1147. /* Establish migration entry for a file page */
  1148. swp_entry_t entry;
  1149. entry = make_migration_entry(page, pte_write(pteval));
  1150. set_pte_at(mm, address, pte, swp_entry_to_pte(entry));
  1151. } else
  1152. dec_mm_counter(mm, MM_FILEPAGES);
  1153. page_remove_rmap(page);
  1154. page_cache_release(page);
  1155. out_unmap:
  1156. pte_unmap_unlock(pte, ptl);
  1157. if (ret != SWAP_FAIL)
  1158. mmu_notifier_invalidate_page(mm, address);
  1159. out:
  1160. return ret;
  1161. out_mlock:
  1162. pte_unmap_unlock(pte, ptl);
  1163. /*
  1164. * We need mmap_sem locking, Otherwise VM_LOCKED check makes
  1165. * unstable result and race. Plus, We can't wait here because
  1166. * we now hold anon_vma->mutex or mapping->i_mmap_mutex.
  1167. * if trylock failed, the page remain in evictable lru and later
  1168. * vmscan could retry to move the page to unevictable lru if the
  1169. * page is actually mlocked.
  1170. */
  1171. if (down_read_trylock(&vma->vm_mm->mmap_sem)) {
  1172. if (vma->vm_flags & VM_LOCKED) {
  1173. mlock_vma_page(page);
  1174. ret = SWAP_MLOCK;
  1175. }
  1176. up_read(&vma->vm_mm->mmap_sem);
  1177. }
  1178. return ret;
  1179. }
  1180. /*
  1181. * objrmap doesn't work for nonlinear VMAs because the assumption that
  1182. * offset-into-file correlates with offset-into-virtual-addresses does not hold.
  1183. * Consequently, given a particular page and its ->index, we cannot locate the
  1184. * ptes which are mapping that page without an exhaustive linear search.
  1185. *
  1186. * So what this code does is a mini "virtual scan" of each nonlinear VMA which
  1187. * maps the file to which the target page belongs. The ->vm_private_data field
  1188. * holds the current cursor into that scan. Successive searches will circulate
  1189. * around the vma's virtual address space.
  1190. *
  1191. * So as more replacement pressure is applied to the pages in a nonlinear VMA,
  1192. * more scanning pressure is placed against them as well. Eventually pages
  1193. * will become fully unmapped and are eligible for eviction.
  1194. *
  1195. * For very sparsely populated VMAs this is a little inefficient - chances are
  1196. * there there won't be many ptes located within the scan cluster. In this case
  1197. * maybe we could scan further - to the end of the pte page, perhaps.
  1198. *
  1199. * Mlocked pages: check VM_LOCKED under mmap_sem held for read, if we can
  1200. * acquire it without blocking. If vma locked, mlock the pages in the cluster,
  1201. * rather than unmapping them. If we encounter the "check_page" that vmscan is
  1202. * trying to unmap, return SWAP_MLOCK, else default SWAP_AGAIN.
  1203. */
  1204. #define CLUSTER_SIZE min(32*PAGE_SIZE, PMD_SIZE)
  1205. #define CLUSTER_MASK (~(CLUSTER_SIZE - 1))
  1206. static int try_to_unmap_cluster(unsigned long cursor, unsigned int *mapcount,
  1207. struct vm_area_struct *vma, struct page *check_page)
  1208. {
  1209. struct mm_struct *mm = vma->vm_mm;
  1210. pmd_t *pmd;
  1211. pte_t *pte;
  1212. pte_t pteval;
  1213. spinlock_t *ptl;
  1214. struct page *page;
  1215. unsigned long address;
  1216. unsigned long mmun_start; /* For mmu_notifiers */
  1217. unsigned long mmun_end; /* For mmu_notifiers */
  1218. unsigned long end;
  1219. int ret = SWAP_AGAIN;
  1220. int locked_vma = 0;
  1221. address = (vma->vm_start + cursor) & CLUSTER_MASK;
  1222. end = address + CLUSTER_SIZE;
  1223. if (address < vma->vm_start)
  1224. address = vma->vm_start;
  1225. if (end > vma->vm_end)
  1226. end = vma->vm_end;
  1227. pmd = mm_find_pmd(mm, address);
  1228. if (!pmd)
  1229. return ret;
  1230. mmun_start = address;
  1231. mmun_end = end;
  1232. mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
  1233. /*
  1234. * If we can acquire the mmap_sem for read, and vma is VM_LOCKED,
  1235. * keep the sem while scanning the cluster for mlocking pages.
  1236. */
  1237. if (down_read_trylock(&vma->vm_mm->mmap_sem)) {
  1238. locked_vma = (vma->vm_flags & VM_LOCKED);
  1239. if (!locked_vma)
  1240. up_read(&vma->vm_mm->mmap_sem); /* don't need it */
  1241. }
  1242. pte = pte_offset_map_lock(mm, pmd, address, &ptl);
  1243. /* Update high watermark before we lower rss */
  1244. update_hiwater_rss(mm);
  1245. for (; address < end; pte++, address += PAGE_SIZE) {
  1246. if (!pte_present(*pte))
  1247. continue;
  1248. page = vm_normal_page(vma, address, *pte);
  1249. BUG_ON(!page || PageAnon(page));
  1250. if (locked_vma) {
  1251. mlock_vma_page(page); /* no-op if already mlocked */
  1252. if (page == check_page)
  1253. ret = SWAP_MLOCK;
  1254. continue; /* don't unmap */
  1255. }
  1256. if (ptep_clear_flush_young_notify(vma, address, pte))
  1257. continue;
  1258. /* Nuke the page table entry. */
  1259. flush_cache_page(vma, address, pte_pfn(*pte));
  1260. pteval = ptep_clear_flush(vma, address, pte);
  1261. /* If nonlinear, store the file page offset in the pte. */
  1262. if (page->index != linear_page_index(vma, address))
  1263. set_pte_at(mm, address, pte, pgoff_to_pte(page->index));
  1264. /* Move the dirty bit to the physical page now the pte is gone. */
  1265. if (pte_dirty(pteval))
  1266. set_page_dirty(page);
  1267. page_remove_rmap(page);
  1268. page_cache_release(page);
  1269. dec_mm_counter(mm, MM_FILEPAGES);
  1270. (*mapcount)--;
  1271. }
  1272. pte_unmap_unlock(pte - 1, ptl);
  1273. mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
  1274. if (locked_vma)
  1275. up_read(&vma->vm_mm->mmap_sem);
  1276. return ret;
  1277. }
  1278. bool is_vma_temporary_stack(struct vm_area_struct *vma)
  1279. {
  1280. int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP);
  1281. if (!maybe_stack)
  1282. return false;
  1283. if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) ==
  1284. VM_STACK_INCOMPLETE_SETUP)
  1285. return true;
  1286. return false;
  1287. }
  1288. /**
  1289. * try_to_unmap_anon - unmap or unlock anonymous page using the object-based
  1290. * rmap method
  1291. * @page: the page to unmap/unlock
  1292. * @flags: action and flags
  1293. *
  1294. * Find all the mappings of a page using the mapping pointer and the vma chains
  1295. * contained in the anon_vma struct it points to.
  1296. *
  1297. * This function is only called from try_to_unmap/try_to_munlock for
  1298. * anonymous pages.
  1299. * When called from try_to_munlock(), the mmap_sem of the mm containing the vma
  1300. * where the page was found will be held for write. So, we won't recheck
  1301. * vm_flags for that VMA. That should be OK, because that vma shouldn't be
  1302. * 'LOCKED.
  1303. */
  1304. static int try_to_unmap_anon(struct page *page, enum ttu_flags flags)
  1305. {
  1306. struct anon_vma *anon_vma;
  1307. pgoff_t pgoff;
  1308. struct anon_vma_chain *avc;
  1309. int ret = SWAP_AGAIN;
  1310. anon_vma = page_lock_anon_vma(page);
  1311. if (!anon_vma)
  1312. return ret;
  1313. pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
  1314. anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff, pgoff) {
  1315. struct vm_area_struct *vma = avc->vma;
  1316. unsigned long address;
  1317. /*
  1318. * During exec, a temporary VMA is setup and later moved.
  1319. * The VMA is moved under the anon_vma lock but not the
  1320. * page tables leading to a race where migration cannot
  1321. * find the migration ptes. Rather than increasing the
  1322. * locking requirements of exec(), migration skips
  1323. * temporary VMAs until after exec() completes.
  1324. */
  1325. if (IS_ENABLED(CONFIG_MIGRATION) && (flags & TTU_MIGRATION) &&
  1326. is_vma_temporary_stack(vma))
  1327. continue;
  1328. address = vma_address(page, vma);
  1329. ret = try_to_unmap_one(page, vma, address, flags);
  1330. if (ret != SWAP_AGAIN || !page_mapped(page))
  1331. break;
  1332. }
  1333. page_unlock_anon_vma(anon_vma);
  1334. return ret;
  1335. }
  1336. /**
  1337. * try_to_unmap_file - unmap/unlock file page using the object-based rmap method
  1338. * @page: the page to unmap/unlock
  1339. * @flags: action and flags
  1340. *
  1341. * Find all the mappings of a page using the mapping pointer and the vma chains
  1342. * contained in the address_space struct it points to.
  1343. *
  1344. * This function is only called from try_to_unmap/try_to_munlock for
  1345. * object-based pages.
  1346. * When called from try_to_munlock(), the mmap_sem of the mm containing the vma
  1347. * where the page was found will be held for write. So, we won't recheck
  1348. * vm_flags for that VMA. That should be OK, because that vma shouldn't be
  1349. * 'LOCKED.
  1350. */
  1351. static int try_to_unmap_file(struct page *page, enum ttu_flags flags)
  1352. {
  1353. struct address_space *mapping = page->mapping;
  1354. pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
  1355. struct vm_area_struct *vma;
  1356. int ret = SWAP_AGAIN;
  1357. unsigned long cursor;
  1358. unsigned long max_nl_cursor = 0;
  1359. unsigned long max_nl_size = 0;
  1360. unsigned int mapcount;
  1361. mutex_lock(&mapping->i_mmap_mutex);
  1362. vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
  1363. unsigned long address = vma_address(page, vma);
  1364. ret = try_to_unmap_one(page, vma, address, flags);
  1365. if (ret != SWAP_AGAIN || !page_mapped(page))
  1366. goto out;
  1367. }
  1368. if (list_empty(&mapping->i_mmap_nonlinear))
  1369. goto out;
  1370. /*
  1371. * We don't bother to try to find the munlocked page in nonlinears.
  1372. * It's costly. Instead, later, page reclaim logic may call
  1373. * try_to_unmap(TTU_MUNLOCK) and recover PG_mlocked lazily.
  1374. */
  1375. if (TTU_ACTION(flags) == TTU_MUNLOCK)
  1376. goto out;
  1377. list_for_each_entry(vma, &mapping->i_mmap_nonlinear,
  1378. shared.nonlinear) {
  1379. cursor = (unsigned long) vma->vm_private_data;
  1380. if (cursor > max_nl_cursor)
  1381. max_nl_cursor = cursor;
  1382. cursor = vma->vm_end - vma->vm_start;
  1383. if (cursor > max_nl_size)
  1384. max_nl_size = cursor;
  1385. }
  1386. if (max_nl_size == 0) { /* all nonlinears locked or reserved ? */
  1387. ret = SWAP_FAIL;
  1388. goto out;
  1389. }
  1390. /*
  1391. * We don't try to search for this page in the nonlinear vmas,
  1392. * and page_referenced wouldn't have found it anyway. Instead
  1393. * just walk the nonlinear vmas trying to age and unmap some.
  1394. * The mapcount of the page we came in with is irrelevant,
  1395. * but even so use it as a guide to how hard we should try?
  1396. */
  1397. mapcount = page_mapcount(page);
  1398. if (!mapcount)
  1399. goto out;
  1400. cond_resched();
  1401. max_nl_size = (max_nl_size + CLUSTER_SIZE - 1) & CLUSTER_MASK;
  1402. if (max_nl_cursor == 0)
  1403. max_nl_cursor = CLUSTER_SIZE;
  1404. do {
  1405. list_for_each_entry(vma, &mapping->i_mmap_nonlinear,
  1406. shared.nonlinear) {
  1407. cursor = (unsigned long) vma->vm_private_data;
  1408. while ( cursor < max_nl_cursor &&
  1409. cursor < vma->vm_end - vma->vm_start) {
  1410. if (try_to_unmap_cluster(cursor, &mapcount,
  1411. vma, page) == SWAP_MLOCK)
  1412. ret = SWAP_MLOCK;
  1413. cursor += CLUSTER_SIZE;
  1414. vma->vm_private_data = (void *) cursor;
  1415. if ((int)mapcount <= 0)
  1416. goto out;
  1417. }
  1418. vma->vm_private_data = (void *) max_nl_cursor;
  1419. }
  1420. cond_resched();
  1421. max_nl_cursor += CLUSTER_SIZE;
  1422. } while (max_nl_cursor <= max_nl_size);
  1423. /*
  1424. * Don't loop forever (perhaps all the remaining pages are
  1425. * in locked vmas). Reset cursor on all unreserved nonlinear
  1426. * vmas, now forgetting on which ones it had fallen behind.
  1427. */
  1428. list_for_each_entry(vma, &mapping->i_mmap_nonlinear, shared.nonlinear)
  1429. vma->vm_private_data = NULL;
  1430. out:
  1431. mutex_unlock(&mapping->i_mmap_mutex);
  1432. return ret;
  1433. }
  1434. /**
  1435. * try_to_unmap - try to remove all page table mappings to a page
  1436. * @page: the page to get unmapped
  1437. * @flags: action and flags
  1438. *
  1439. * Tries to remove all the page table entries which are mapping this
  1440. * page, used in the pageout path. Caller must hold the page lock.
  1441. * Return values are:
  1442. *
  1443. * SWAP_SUCCESS - we succeeded in removing all mappings
  1444. * SWAP_AGAIN - we missed a mapping, try again later
  1445. * SWAP_FAIL - the page is unswappable
  1446. * SWAP_MLOCK - page is mlocked.
  1447. */
  1448. int try_to_unmap(struct page *page, enum ttu_flags flags)
  1449. {
  1450. int ret;
  1451. BUG_ON(!PageLocked(page));
  1452. VM_BUG_ON(!PageHuge(page) && PageTransHuge(page));
  1453. if (unlikely(PageKsm(page)))
  1454. ret = try_to_unmap_ksm(page, flags);
  1455. else if (PageAnon(page))
  1456. ret = try_to_unmap_anon(page, flags);
  1457. else
  1458. ret = try_to_unmap_file(page, flags);
  1459. if (ret != SWAP_MLOCK && !page_mapped(page))
  1460. ret = SWAP_SUCCESS;
  1461. return ret;
  1462. }
  1463. /**
  1464. * try_to_munlock - try to munlock a page
  1465. * @page: the page to be munlocked
  1466. *
  1467. * Called from munlock code. Checks all of the VMAs mapping the page
  1468. * to make sure nobody else has this page mlocked. The page will be
  1469. * returned with PG_mlocked cleared if no other vmas have it mlocked.
  1470. *
  1471. * Return values are:
  1472. *
  1473. * SWAP_AGAIN - no vma is holding page mlocked, or,
  1474. * SWAP_AGAIN - page mapped in mlocked vma -- couldn't acquire mmap sem
  1475. * SWAP_FAIL - page cannot be located at present
  1476. * SWAP_MLOCK - page is now mlocked.
  1477. */
  1478. int try_to_munlock(struct page *page)
  1479. {
  1480. VM_BUG_ON(!PageLocked(page) || PageLRU(page));
  1481. if (unlikely(PageKsm(page)))
  1482. return try_to_unmap_ksm(page, TTU_MUNLOCK);
  1483. else if (PageAnon(page))
  1484. return try_to_unmap_anon(page, TTU_MUNLOCK);
  1485. else
  1486. return try_to_unmap_file(page, TTU_MUNLOCK);
  1487. }
  1488. void __put_anon_vma(struct anon_vma *anon_vma)
  1489. {
  1490. struct anon_vma *root = anon_vma->root;
  1491. if (root != anon_vma && atomic_dec_and_test(&root->refcount))
  1492. anon_vma_free(root);
  1493. anon_vma_free(anon_vma);
  1494. }
  1495. #ifdef CONFIG_MIGRATION
  1496. /*
  1497. * rmap_walk() and its helpers rmap_walk_anon() and rmap_walk_file():
  1498. * Called by migrate.c to remove migration ptes, but might be used more later.
  1499. */
  1500. static int rmap_walk_anon(struct page *page, int (*rmap_one)(struct page *,
  1501. struct vm_area_struct *, unsigned long, void *), void *arg)
  1502. {
  1503. struct anon_vma *anon_vma;
  1504. pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
  1505. struct anon_vma_chain *avc;
  1506. int ret = SWAP_AGAIN;
  1507. /*
  1508. * Note: remove_migration_ptes() cannot use page_lock_anon_vma()
  1509. * because that depends on page_mapped(); but not all its usages
  1510. * are holding mmap_sem. Users without mmap_sem are required to
  1511. * take a reference count to prevent the anon_vma disappearing
  1512. */
  1513. anon_vma = page_anon_vma(page);
  1514. if (!anon_vma)
  1515. return ret;
  1516. anon_vma_lock(anon_vma);
  1517. anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff, pgoff) {
  1518. struct vm_area_struct *vma = avc->vma;
  1519. unsigned long address = vma_address(page, vma);
  1520. ret = rmap_one(page, vma, address, arg);
  1521. if (ret != SWAP_AGAIN)
  1522. break;
  1523. }
  1524. anon_vma_unlock(anon_vma);
  1525. return ret;
  1526. }
  1527. static int rmap_walk_file(struct page *page, int (*rmap_one)(struct page *,
  1528. struct vm_area_struct *, unsigned long, void *), void *arg)
  1529. {
  1530. struct address_space *mapping = page->mapping;
  1531. pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
  1532. struct vm_area_struct *vma;
  1533. int ret = SWAP_AGAIN;
  1534. if (!mapping)
  1535. return ret;
  1536. mutex_lock(&mapping->i_mmap_mutex);
  1537. vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
  1538. unsigned long address = vma_address(page, vma);
  1539. ret = rmap_one(page, vma, address, arg);
  1540. if (ret != SWAP_AGAIN)
  1541. break;
  1542. }
  1543. /*
  1544. * No nonlinear handling: being always shared, nonlinear vmas
  1545. * never contain migration ptes. Decide what to do about this
  1546. * limitation to linear when we need rmap_walk() on nonlinear.
  1547. */
  1548. mutex_unlock(&mapping->i_mmap_mutex);
  1549. return ret;
  1550. }
  1551. int rmap_walk(struct page *page, int (*rmap_one)(struct page *,
  1552. struct vm_area_struct *, unsigned long, void *), void *arg)
  1553. {
  1554. VM_BUG_ON(!PageLocked(page));
  1555. if (unlikely(PageKsm(page)))
  1556. return rmap_walk_ksm(page, rmap_one, arg);
  1557. else if (PageAnon(page))
  1558. return rmap_walk_anon(page, rmap_one, arg);
  1559. else
  1560. return rmap_walk_file(page, rmap_one, arg);
  1561. }
  1562. #endif /* CONFIG_MIGRATION */
  1563. #ifdef CONFIG_HUGETLB_PAGE
  1564. /*
  1565. * The following three functions are for anonymous (private mapped) hugepages.
  1566. * Unlike common anonymous pages, anonymous hugepages have no accounting code
  1567. * and no lru code, because we handle hugepages differently from common pages.
  1568. */
  1569. static void __hugepage_set_anon_rmap(struct page *page,
  1570. struct vm_area_struct *vma, unsigned long address, int exclusive)
  1571. {
  1572. struct anon_vma *anon_vma = vma->anon_vma;
  1573. BUG_ON(!anon_vma);
  1574. if (PageAnon(page))
  1575. return;
  1576. if (!exclusive)
  1577. anon_vma = anon_vma->root;
  1578. anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
  1579. page->mapping = (struct address_space *) anon_vma;
  1580. page->index = linear_page_index(vma, address);
  1581. }
  1582. void hugepage_add_anon_rmap(struct page *page,
  1583. struct vm_area_struct *vma, unsigned long address)
  1584. {
  1585. struct anon_vma *anon_vma = vma->anon_vma;
  1586. int first;
  1587. BUG_ON(!PageLocked(page));
  1588. BUG_ON(!anon_vma);
  1589. /* address might be in next vma when migration races vma_adjust */
  1590. first = atomic_inc_and_test(&page->_mapcount);
  1591. if (first)
  1592. __hugepage_set_anon_rmap(page, vma, address, 0);
  1593. }
  1594. void hugepage_add_new_anon_rmap(struct page *page,
  1595. struct vm_area_struct *vma, unsigned long address)
  1596. {
  1597. BUG_ON(address < vma->vm_start || address >= vma->vm_end);
  1598. atomic_set(&page->_mapcount, 0);
  1599. __hugepage_set_anon_rmap(page, vma, address, 1);
  1600. }
  1601. #endif /* CONFIG_HUGETLB_PAGE */