huge_memory.c 77 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929
  1. /*
  2. * Copyright (C) 2009 Red Hat, Inc.
  3. *
  4. * This work is licensed under the terms of the GNU GPL, version 2. See
  5. * the COPYING file in the top-level directory.
  6. */
  7. #include <linux/mm.h>
  8. #include <linux/sched.h>
  9. #include <linux/highmem.h>
  10. #include <linux/hugetlb.h>
  11. #include <linux/mmu_notifier.h>
  12. #include <linux/rmap.h>
  13. #include <linux/swap.h>
  14. #include <linux/shrinker.h>
  15. #include <linux/mm_inline.h>
  16. #include <linux/kthread.h>
  17. #include <linux/khugepaged.h>
  18. #include <linux/freezer.h>
  19. #include <linux/mman.h>
  20. #include <linux/pagemap.h>
  21. #include <linux/migrate.h>
  22. #include <linux/hashtable.h>
  23. #include <asm/tlb.h>
  24. #include <asm/pgalloc.h>
  25. #include "internal.h"
  26. /*
  27. * By default transparent hugepage support is disabled in order that avoid
  28. * to risk increase the memory footprint of applications without a guaranteed
  29. * benefit. When transparent hugepage support is enabled, is for all mappings,
  30. * and khugepaged scans all mappings.
  31. * Defrag is invoked by khugepaged hugepage allocations and by page faults
  32. * for all hugepage allocations.
  33. */
  34. unsigned long transparent_hugepage_flags __read_mostly =
  35. #ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS
  36. (1<<TRANSPARENT_HUGEPAGE_FLAG)|
  37. #endif
  38. #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE
  39. (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)|
  40. #endif
  41. (1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG)|
  42. (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)|
  43. (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
  44. /* default scan 8*512 pte (or vmas) every 30 second */
  45. static unsigned int khugepaged_pages_to_scan __read_mostly = HPAGE_PMD_NR*8;
  46. static unsigned int khugepaged_pages_collapsed;
  47. static unsigned int khugepaged_full_scans;
  48. static unsigned int khugepaged_scan_sleep_millisecs __read_mostly = 10000;
  49. /* during fragmentation poll the hugepage allocator once every minute */
  50. static unsigned int khugepaged_alloc_sleep_millisecs __read_mostly = 60000;
  51. static struct task_struct *khugepaged_thread __read_mostly;
  52. static DEFINE_MUTEX(khugepaged_mutex);
  53. static DEFINE_SPINLOCK(khugepaged_mm_lock);
  54. static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait);
  55. /*
  56. * default collapse hugepages if there is at least one pte mapped like
  57. * it would have happened if the vma was large enough during page
  58. * fault.
  59. */
  60. static unsigned int khugepaged_max_ptes_none __read_mostly = HPAGE_PMD_NR-1;
  61. static int khugepaged(void *none);
  62. static int khugepaged_slab_init(void);
  63. #define MM_SLOTS_HASH_BITS 10
  64. static __read_mostly DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS);
  65. static struct kmem_cache *mm_slot_cache __read_mostly;
  66. /**
  67. * struct mm_slot - hash lookup from mm to mm_slot
  68. * @hash: hash collision list
  69. * @mm_node: khugepaged scan list headed in khugepaged_scan.mm_head
  70. * @mm: the mm that this information is valid for
  71. */
  72. struct mm_slot {
  73. struct hlist_node hash;
  74. struct list_head mm_node;
  75. struct mm_struct *mm;
  76. };
  77. /**
  78. * struct khugepaged_scan - cursor for scanning
  79. * @mm_head: the head of the mm list to scan
  80. * @mm_slot: the current mm_slot we are scanning
  81. * @address: the next address inside that to be scanned
  82. *
  83. * There is only the one khugepaged_scan instance of this cursor structure.
  84. */
  85. struct khugepaged_scan {
  86. struct list_head mm_head;
  87. struct mm_slot *mm_slot;
  88. unsigned long address;
  89. };
  90. static struct khugepaged_scan khugepaged_scan = {
  91. .mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head),
  92. };
  93. static int set_recommended_min_free_kbytes(void)
  94. {
  95. struct zone *zone;
  96. int nr_zones = 0;
  97. unsigned long recommended_min;
  98. if (!khugepaged_enabled())
  99. return 0;
  100. for_each_populated_zone(zone)
  101. nr_zones++;
  102. /* Make sure at least 2 hugepages are free for MIGRATE_RESERVE */
  103. recommended_min = pageblock_nr_pages * nr_zones * 2;
  104. /*
  105. * Make sure that on average at least two pageblocks are almost free
  106. * of another type, one for a migratetype to fall back to and a
  107. * second to avoid subsequent fallbacks of other types There are 3
  108. * MIGRATE_TYPES we care about.
  109. */
  110. recommended_min += pageblock_nr_pages * nr_zones *
  111. MIGRATE_PCPTYPES * MIGRATE_PCPTYPES;
  112. /* don't ever allow to reserve more than 5% of the lowmem */
  113. recommended_min = min(recommended_min,
  114. (unsigned long) nr_free_buffer_pages() / 20);
  115. recommended_min <<= (PAGE_SHIFT-10);
  116. if (recommended_min > min_free_kbytes)
  117. min_free_kbytes = recommended_min;
  118. setup_per_zone_wmarks();
  119. return 0;
  120. }
  121. late_initcall(set_recommended_min_free_kbytes);
  122. static int start_khugepaged(void)
  123. {
  124. int err = 0;
  125. if (khugepaged_enabled()) {
  126. if (!khugepaged_thread)
  127. khugepaged_thread = kthread_run(khugepaged, NULL,
  128. "khugepaged");
  129. if (unlikely(IS_ERR(khugepaged_thread))) {
  130. printk(KERN_ERR
  131. "khugepaged: kthread_run(khugepaged) failed\n");
  132. err = PTR_ERR(khugepaged_thread);
  133. khugepaged_thread = NULL;
  134. }
  135. if (!list_empty(&khugepaged_scan.mm_head))
  136. wake_up_interruptible(&khugepaged_wait);
  137. set_recommended_min_free_kbytes();
  138. } else if (khugepaged_thread) {
  139. kthread_stop(khugepaged_thread);
  140. khugepaged_thread = NULL;
  141. }
  142. return err;
  143. }
  144. static atomic_t huge_zero_refcount;
  145. static struct page *huge_zero_page __read_mostly;
  146. static inline bool is_huge_zero_page(struct page *page)
  147. {
  148. return ACCESS_ONCE(huge_zero_page) == page;
  149. }
  150. static inline bool is_huge_zero_pmd(pmd_t pmd)
  151. {
  152. return is_huge_zero_page(pmd_page(pmd));
  153. }
  154. static struct page *get_huge_zero_page(void)
  155. {
  156. struct page *zero_page;
  157. retry:
  158. if (likely(atomic_inc_not_zero(&huge_zero_refcount)))
  159. return ACCESS_ONCE(huge_zero_page);
  160. zero_page = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE,
  161. HPAGE_PMD_ORDER);
  162. if (!zero_page) {
  163. count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED);
  164. return NULL;
  165. }
  166. count_vm_event(THP_ZERO_PAGE_ALLOC);
  167. preempt_disable();
  168. if (cmpxchg(&huge_zero_page, NULL, zero_page)) {
  169. preempt_enable();
  170. __free_page(zero_page);
  171. goto retry;
  172. }
  173. /* We take additional reference here. It will be put back by shrinker */
  174. atomic_set(&huge_zero_refcount, 2);
  175. preempt_enable();
  176. return ACCESS_ONCE(huge_zero_page);
  177. }
  178. static void put_huge_zero_page(void)
  179. {
  180. /*
  181. * Counter should never go to zero here. Only shrinker can put
  182. * last reference.
  183. */
  184. BUG_ON(atomic_dec_and_test(&huge_zero_refcount));
  185. }
  186. static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink,
  187. struct shrink_control *sc)
  188. {
  189. /* we can free zero page only if last reference remains */
  190. return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0;
  191. }
  192. static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink,
  193. struct shrink_control *sc)
  194. {
  195. if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) {
  196. struct page *zero_page = xchg(&huge_zero_page, NULL);
  197. BUG_ON(zero_page == NULL);
  198. __free_page(zero_page);
  199. return HPAGE_PMD_NR;
  200. }
  201. return 0;
  202. }
  203. static struct shrinker huge_zero_page_shrinker = {
  204. .count_objects = shrink_huge_zero_page_count,
  205. .scan_objects = shrink_huge_zero_page_scan,
  206. .seeks = DEFAULT_SEEKS,
  207. };
  208. #ifdef CONFIG_SYSFS
  209. static ssize_t double_flag_show(struct kobject *kobj,
  210. struct kobj_attribute *attr, char *buf,
  211. enum transparent_hugepage_flag enabled,
  212. enum transparent_hugepage_flag req_madv)
  213. {
  214. if (test_bit(enabled, &transparent_hugepage_flags)) {
  215. VM_BUG_ON(test_bit(req_madv, &transparent_hugepage_flags));
  216. return sprintf(buf, "[always] madvise never\n");
  217. } else if (test_bit(req_madv, &transparent_hugepage_flags))
  218. return sprintf(buf, "always [madvise] never\n");
  219. else
  220. return sprintf(buf, "always madvise [never]\n");
  221. }
  222. static ssize_t double_flag_store(struct kobject *kobj,
  223. struct kobj_attribute *attr,
  224. const char *buf, size_t count,
  225. enum transparent_hugepage_flag enabled,
  226. enum transparent_hugepage_flag req_madv)
  227. {
  228. if (!memcmp("always", buf,
  229. min(sizeof("always")-1, count))) {
  230. set_bit(enabled, &transparent_hugepage_flags);
  231. clear_bit(req_madv, &transparent_hugepage_flags);
  232. } else if (!memcmp("madvise", buf,
  233. min(sizeof("madvise")-1, count))) {
  234. clear_bit(enabled, &transparent_hugepage_flags);
  235. set_bit(req_madv, &transparent_hugepage_flags);
  236. } else if (!memcmp("never", buf,
  237. min(sizeof("never")-1, count))) {
  238. clear_bit(enabled, &transparent_hugepage_flags);
  239. clear_bit(req_madv, &transparent_hugepage_flags);
  240. } else
  241. return -EINVAL;
  242. return count;
  243. }
  244. static ssize_t enabled_show(struct kobject *kobj,
  245. struct kobj_attribute *attr, char *buf)
  246. {
  247. return double_flag_show(kobj, attr, buf,
  248. TRANSPARENT_HUGEPAGE_FLAG,
  249. TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG);
  250. }
  251. static ssize_t enabled_store(struct kobject *kobj,
  252. struct kobj_attribute *attr,
  253. const char *buf, size_t count)
  254. {
  255. ssize_t ret;
  256. ret = double_flag_store(kobj, attr, buf, count,
  257. TRANSPARENT_HUGEPAGE_FLAG,
  258. TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG);
  259. if (ret > 0) {
  260. int err;
  261. mutex_lock(&khugepaged_mutex);
  262. err = start_khugepaged();
  263. mutex_unlock(&khugepaged_mutex);
  264. if (err)
  265. ret = err;
  266. }
  267. return ret;
  268. }
  269. static struct kobj_attribute enabled_attr =
  270. __ATTR(enabled, 0644, enabled_show, enabled_store);
  271. static ssize_t single_flag_show(struct kobject *kobj,
  272. struct kobj_attribute *attr, char *buf,
  273. enum transparent_hugepage_flag flag)
  274. {
  275. return sprintf(buf, "%d\n",
  276. !!test_bit(flag, &transparent_hugepage_flags));
  277. }
  278. static ssize_t single_flag_store(struct kobject *kobj,
  279. struct kobj_attribute *attr,
  280. const char *buf, size_t count,
  281. enum transparent_hugepage_flag flag)
  282. {
  283. unsigned long value;
  284. int ret;
  285. ret = kstrtoul(buf, 10, &value);
  286. if (ret < 0)
  287. return ret;
  288. if (value > 1)
  289. return -EINVAL;
  290. if (value)
  291. set_bit(flag, &transparent_hugepage_flags);
  292. else
  293. clear_bit(flag, &transparent_hugepage_flags);
  294. return count;
  295. }
  296. /*
  297. * Currently defrag only disables __GFP_NOWAIT for allocation. A blind
  298. * __GFP_REPEAT is too aggressive, it's never worth swapping tons of
  299. * memory just to allocate one more hugepage.
  300. */
  301. static ssize_t defrag_show(struct kobject *kobj,
  302. struct kobj_attribute *attr, char *buf)
  303. {
  304. return double_flag_show(kobj, attr, buf,
  305. TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,
  306. TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG);
  307. }
  308. static ssize_t defrag_store(struct kobject *kobj,
  309. struct kobj_attribute *attr,
  310. const char *buf, size_t count)
  311. {
  312. return double_flag_store(kobj, attr, buf, count,
  313. TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,
  314. TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG);
  315. }
  316. static struct kobj_attribute defrag_attr =
  317. __ATTR(defrag, 0644, defrag_show, defrag_store);
  318. static ssize_t use_zero_page_show(struct kobject *kobj,
  319. struct kobj_attribute *attr, char *buf)
  320. {
  321. return single_flag_show(kobj, attr, buf,
  322. TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
  323. }
  324. static ssize_t use_zero_page_store(struct kobject *kobj,
  325. struct kobj_attribute *attr, const char *buf, size_t count)
  326. {
  327. return single_flag_store(kobj, attr, buf, count,
  328. TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
  329. }
  330. static struct kobj_attribute use_zero_page_attr =
  331. __ATTR(use_zero_page, 0644, use_zero_page_show, use_zero_page_store);
  332. #ifdef CONFIG_DEBUG_VM
  333. static ssize_t debug_cow_show(struct kobject *kobj,
  334. struct kobj_attribute *attr, char *buf)
  335. {
  336. return single_flag_show(kobj, attr, buf,
  337. TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
  338. }
  339. static ssize_t debug_cow_store(struct kobject *kobj,
  340. struct kobj_attribute *attr,
  341. const char *buf, size_t count)
  342. {
  343. return single_flag_store(kobj, attr, buf, count,
  344. TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
  345. }
  346. static struct kobj_attribute debug_cow_attr =
  347. __ATTR(debug_cow, 0644, debug_cow_show, debug_cow_store);
  348. #endif /* CONFIG_DEBUG_VM */
  349. static struct attribute *hugepage_attr[] = {
  350. &enabled_attr.attr,
  351. &defrag_attr.attr,
  352. &use_zero_page_attr.attr,
  353. #ifdef CONFIG_DEBUG_VM
  354. &debug_cow_attr.attr,
  355. #endif
  356. NULL,
  357. };
  358. static struct attribute_group hugepage_attr_group = {
  359. .attrs = hugepage_attr,
  360. };
  361. static ssize_t scan_sleep_millisecs_show(struct kobject *kobj,
  362. struct kobj_attribute *attr,
  363. char *buf)
  364. {
  365. return sprintf(buf, "%u\n", khugepaged_scan_sleep_millisecs);
  366. }
  367. static ssize_t scan_sleep_millisecs_store(struct kobject *kobj,
  368. struct kobj_attribute *attr,
  369. const char *buf, size_t count)
  370. {
  371. unsigned long msecs;
  372. int err;
  373. err = kstrtoul(buf, 10, &msecs);
  374. if (err || msecs > UINT_MAX)
  375. return -EINVAL;
  376. khugepaged_scan_sleep_millisecs = msecs;
  377. wake_up_interruptible(&khugepaged_wait);
  378. return count;
  379. }
  380. static struct kobj_attribute scan_sleep_millisecs_attr =
  381. __ATTR(scan_sleep_millisecs, 0644, scan_sleep_millisecs_show,
  382. scan_sleep_millisecs_store);
  383. static ssize_t alloc_sleep_millisecs_show(struct kobject *kobj,
  384. struct kobj_attribute *attr,
  385. char *buf)
  386. {
  387. return sprintf(buf, "%u\n", khugepaged_alloc_sleep_millisecs);
  388. }
  389. static ssize_t alloc_sleep_millisecs_store(struct kobject *kobj,
  390. struct kobj_attribute *attr,
  391. const char *buf, size_t count)
  392. {
  393. unsigned long msecs;
  394. int err;
  395. err = kstrtoul(buf, 10, &msecs);
  396. if (err || msecs > UINT_MAX)
  397. return -EINVAL;
  398. khugepaged_alloc_sleep_millisecs = msecs;
  399. wake_up_interruptible(&khugepaged_wait);
  400. return count;
  401. }
  402. static struct kobj_attribute alloc_sleep_millisecs_attr =
  403. __ATTR(alloc_sleep_millisecs, 0644, alloc_sleep_millisecs_show,
  404. alloc_sleep_millisecs_store);
  405. static ssize_t pages_to_scan_show(struct kobject *kobj,
  406. struct kobj_attribute *attr,
  407. char *buf)
  408. {
  409. return sprintf(buf, "%u\n", khugepaged_pages_to_scan);
  410. }
  411. static ssize_t pages_to_scan_store(struct kobject *kobj,
  412. struct kobj_attribute *attr,
  413. const char *buf, size_t count)
  414. {
  415. int err;
  416. unsigned long pages;
  417. err = kstrtoul(buf, 10, &pages);
  418. if (err || !pages || pages > UINT_MAX)
  419. return -EINVAL;
  420. khugepaged_pages_to_scan = pages;
  421. return count;
  422. }
  423. static struct kobj_attribute pages_to_scan_attr =
  424. __ATTR(pages_to_scan, 0644, pages_to_scan_show,
  425. pages_to_scan_store);
  426. static ssize_t pages_collapsed_show(struct kobject *kobj,
  427. struct kobj_attribute *attr,
  428. char *buf)
  429. {
  430. return sprintf(buf, "%u\n", khugepaged_pages_collapsed);
  431. }
  432. static struct kobj_attribute pages_collapsed_attr =
  433. __ATTR_RO(pages_collapsed);
  434. static ssize_t full_scans_show(struct kobject *kobj,
  435. struct kobj_attribute *attr,
  436. char *buf)
  437. {
  438. return sprintf(buf, "%u\n", khugepaged_full_scans);
  439. }
  440. static struct kobj_attribute full_scans_attr =
  441. __ATTR_RO(full_scans);
  442. static ssize_t khugepaged_defrag_show(struct kobject *kobj,
  443. struct kobj_attribute *attr, char *buf)
  444. {
  445. return single_flag_show(kobj, attr, buf,
  446. TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG);
  447. }
  448. static ssize_t khugepaged_defrag_store(struct kobject *kobj,
  449. struct kobj_attribute *attr,
  450. const char *buf, size_t count)
  451. {
  452. return single_flag_store(kobj, attr, buf, count,
  453. TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG);
  454. }
  455. static struct kobj_attribute khugepaged_defrag_attr =
  456. __ATTR(defrag, 0644, khugepaged_defrag_show,
  457. khugepaged_defrag_store);
  458. /*
  459. * max_ptes_none controls if khugepaged should collapse hugepages over
  460. * any unmapped ptes in turn potentially increasing the memory
  461. * footprint of the vmas. When max_ptes_none is 0 khugepaged will not
  462. * reduce the available free memory in the system as it
  463. * runs. Increasing max_ptes_none will instead potentially reduce the
  464. * free memory in the system during the khugepaged scan.
  465. */
  466. static ssize_t khugepaged_max_ptes_none_show(struct kobject *kobj,
  467. struct kobj_attribute *attr,
  468. char *buf)
  469. {
  470. return sprintf(buf, "%u\n", khugepaged_max_ptes_none);
  471. }
  472. static ssize_t khugepaged_max_ptes_none_store(struct kobject *kobj,
  473. struct kobj_attribute *attr,
  474. const char *buf, size_t count)
  475. {
  476. int err;
  477. unsigned long max_ptes_none;
  478. err = kstrtoul(buf, 10, &max_ptes_none);
  479. if (err || max_ptes_none > HPAGE_PMD_NR-1)
  480. return -EINVAL;
  481. khugepaged_max_ptes_none = max_ptes_none;
  482. return count;
  483. }
  484. static struct kobj_attribute khugepaged_max_ptes_none_attr =
  485. __ATTR(max_ptes_none, 0644, khugepaged_max_ptes_none_show,
  486. khugepaged_max_ptes_none_store);
  487. static struct attribute *khugepaged_attr[] = {
  488. &khugepaged_defrag_attr.attr,
  489. &khugepaged_max_ptes_none_attr.attr,
  490. &pages_to_scan_attr.attr,
  491. &pages_collapsed_attr.attr,
  492. &full_scans_attr.attr,
  493. &scan_sleep_millisecs_attr.attr,
  494. &alloc_sleep_millisecs_attr.attr,
  495. NULL,
  496. };
  497. static struct attribute_group khugepaged_attr_group = {
  498. .attrs = khugepaged_attr,
  499. .name = "khugepaged",
  500. };
  501. static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj)
  502. {
  503. int err;
  504. *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj);
  505. if (unlikely(!*hugepage_kobj)) {
  506. printk(KERN_ERR "hugepage: failed to create transparent hugepage kobject\n");
  507. return -ENOMEM;
  508. }
  509. err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group);
  510. if (err) {
  511. printk(KERN_ERR "hugepage: failed to register transparent hugepage group\n");
  512. goto delete_obj;
  513. }
  514. err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group);
  515. if (err) {
  516. printk(KERN_ERR "hugepage: failed to register transparent hugepage group\n");
  517. goto remove_hp_group;
  518. }
  519. return 0;
  520. remove_hp_group:
  521. sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group);
  522. delete_obj:
  523. kobject_put(*hugepage_kobj);
  524. return err;
  525. }
  526. static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj)
  527. {
  528. sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group);
  529. sysfs_remove_group(hugepage_kobj, &hugepage_attr_group);
  530. kobject_put(hugepage_kobj);
  531. }
  532. #else
  533. static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj)
  534. {
  535. return 0;
  536. }
  537. static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj)
  538. {
  539. }
  540. #endif /* CONFIG_SYSFS */
  541. static int __init hugepage_init(void)
  542. {
  543. int err;
  544. struct kobject *hugepage_kobj;
  545. if (!has_transparent_hugepage()) {
  546. transparent_hugepage_flags = 0;
  547. return -EINVAL;
  548. }
  549. err = hugepage_init_sysfs(&hugepage_kobj);
  550. if (err)
  551. return err;
  552. err = khugepaged_slab_init();
  553. if (err)
  554. goto out;
  555. register_shrinker(&huge_zero_page_shrinker);
  556. /*
  557. * By default disable transparent hugepages on smaller systems,
  558. * where the extra memory used could hurt more than TLB overhead
  559. * is likely to save. The admin can still enable it through /sys.
  560. */
  561. if (totalram_pages < (512 << (20 - PAGE_SHIFT)))
  562. transparent_hugepage_flags = 0;
  563. start_khugepaged();
  564. return 0;
  565. out:
  566. hugepage_exit_sysfs(hugepage_kobj);
  567. return err;
  568. }
  569. module_init(hugepage_init)
  570. static int __init setup_transparent_hugepage(char *str)
  571. {
  572. int ret = 0;
  573. if (!str)
  574. goto out;
  575. if (!strcmp(str, "always")) {
  576. set_bit(TRANSPARENT_HUGEPAGE_FLAG,
  577. &transparent_hugepage_flags);
  578. clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
  579. &transparent_hugepage_flags);
  580. ret = 1;
  581. } else if (!strcmp(str, "madvise")) {
  582. clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
  583. &transparent_hugepage_flags);
  584. set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
  585. &transparent_hugepage_flags);
  586. ret = 1;
  587. } else if (!strcmp(str, "never")) {
  588. clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
  589. &transparent_hugepage_flags);
  590. clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
  591. &transparent_hugepage_flags);
  592. ret = 1;
  593. }
  594. out:
  595. if (!ret)
  596. printk(KERN_WARNING
  597. "transparent_hugepage= cannot parse, ignored\n");
  598. return ret;
  599. }
  600. __setup("transparent_hugepage=", setup_transparent_hugepage);
  601. pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
  602. {
  603. if (likely(vma->vm_flags & VM_WRITE))
  604. pmd = pmd_mkwrite(pmd);
  605. return pmd;
  606. }
  607. static inline pmd_t mk_huge_pmd(struct page *page, pgprot_t prot)
  608. {
  609. pmd_t entry;
  610. entry = mk_pmd(page, prot);
  611. entry = pmd_mkhuge(entry);
  612. return entry;
  613. }
  614. static int __do_huge_pmd_anonymous_page(struct mm_struct *mm,
  615. struct vm_area_struct *vma,
  616. unsigned long haddr, pmd_t *pmd,
  617. struct page *page)
  618. {
  619. pgtable_t pgtable;
  620. spinlock_t *ptl;
  621. VM_BUG_ON(!PageCompound(page));
  622. pgtable = pte_alloc_one(mm, haddr);
  623. if (unlikely(!pgtable))
  624. return VM_FAULT_OOM;
  625. clear_huge_page(page, haddr, HPAGE_PMD_NR);
  626. /*
  627. * The memory barrier inside __SetPageUptodate makes sure that
  628. * clear_huge_page writes become visible before the set_pmd_at()
  629. * write.
  630. */
  631. __SetPageUptodate(page);
  632. ptl = pmd_lock(mm, pmd);
  633. if (unlikely(!pmd_none(*pmd))) {
  634. spin_unlock(ptl);
  635. mem_cgroup_uncharge_page(page);
  636. put_page(page);
  637. pte_free(mm, pgtable);
  638. } else {
  639. pmd_t entry;
  640. entry = mk_huge_pmd(page, vma->vm_page_prot);
  641. entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
  642. page_add_new_anon_rmap(page, vma, haddr);
  643. pgtable_trans_huge_deposit(mm, pmd, pgtable);
  644. set_pmd_at(mm, haddr, pmd, entry);
  645. add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR);
  646. atomic_long_inc(&mm->nr_ptes);
  647. spin_unlock(ptl);
  648. }
  649. return 0;
  650. }
  651. static inline gfp_t alloc_hugepage_gfpmask(int defrag, gfp_t extra_gfp)
  652. {
  653. return (GFP_TRANSHUGE & ~(defrag ? 0 : __GFP_WAIT)) | extra_gfp;
  654. }
  655. static inline struct page *alloc_hugepage_vma(int defrag,
  656. struct vm_area_struct *vma,
  657. unsigned long haddr, int nd,
  658. gfp_t extra_gfp)
  659. {
  660. return alloc_pages_vma(alloc_hugepage_gfpmask(defrag, extra_gfp),
  661. HPAGE_PMD_ORDER, vma, haddr, nd);
  662. }
  663. /* Caller must hold page table lock. */
  664. static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm,
  665. struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,
  666. struct page *zero_page)
  667. {
  668. pmd_t entry;
  669. if (!pmd_none(*pmd))
  670. return false;
  671. entry = mk_pmd(zero_page, vma->vm_page_prot);
  672. entry = pmd_wrprotect(entry);
  673. entry = pmd_mkhuge(entry);
  674. pgtable_trans_huge_deposit(mm, pmd, pgtable);
  675. set_pmd_at(mm, haddr, pmd, entry);
  676. atomic_long_inc(&mm->nr_ptes);
  677. return true;
  678. }
  679. int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
  680. unsigned long address, pmd_t *pmd,
  681. unsigned int flags)
  682. {
  683. struct page *page;
  684. unsigned long haddr = address & HPAGE_PMD_MASK;
  685. if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end)
  686. return VM_FAULT_FALLBACK;
  687. if (unlikely(anon_vma_prepare(vma)))
  688. return VM_FAULT_OOM;
  689. if (unlikely(khugepaged_enter(vma)))
  690. return VM_FAULT_OOM;
  691. if (!(flags & FAULT_FLAG_WRITE) &&
  692. transparent_hugepage_use_zero_page()) {
  693. spinlock_t *ptl;
  694. pgtable_t pgtable;
  695. struct page *zero_page;
  696. bool set;
  697. pgtable = pte_alloc_one(mm, haddr);
  698. if (unlikely(!pgtable))
  699. return VM_FAULT_OOM;
  700. zero_page = get_huge_zero_page();
  701. if (unlikely(!zero_page)) {
  702. pte_free(mm, pgtable);
  703. count_vm_event(THP_FAULT_FALLBACK);
  704. return VM_FAULT_FALLBACK;
  705. }
  706. ptl = pmd_lock(mm, pmd);
  707. set = set_huge_zero_page(pgtable, mm, vma, haddr, pmd,
  708. zero_page);
  709. spin_unlock(ptl);
  710. if (!set) {
  711. pte_free(mm, pgtable);
  712. put_huge_zero_page();
  713. }
  714. return 0;
  715. }
  716. page = alloc_hugepage_vma(transparent_hugepage_defrag(vma),
  717. vma, haddr, numa_node_id(), 0);
  718. if (unlikely(!page)) {
  719. count_vm_event(THP_FAULT_FALLBACK);
  720. return VM_FAULT_FALLBACK;
  721. }
  722. if (unlikely(mem_cgroup_newpage_charge(page, mm, GFP_KERNEL))) {
  723. put_page(page);
  724. count_vm_event(THP_FAULT_FALLBACK);
  725. return VM_FAULT_FALLBACK;
  726. }
  727. if (unlikely(__do_huge_pmd_anonymous_page(mm, vma, haddr, pmd, page))) {
  728. mem_cgroup_uncharge_page(page);
  729. put_page(page);
  730. count_vm_event(THP_FAULT_FALLBACK);
  731. return VM_FAULT_FALLBACK;
  732. }
  733. count_vm_event(THP_FAULT_ALLOC);
  734. return 0;
  735. }
  736. int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  737. pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
  738. struct vm_area_struct *vma)
  739. {
  740. spinlock_t *dst_ptl, *src_ptl;
  741. struct page *src_page;
  742. pmd_t pmd;
  743. pgtable_t pgtable;
  744. int ret;
  745. ret = -ENOMEM;
  746. pgtable = pte_alloc_one(dst_mm, addr);
  747. if (unlikely(!pgtable))
  748. goto out;
  749. dst_ptl = pmd_lock(dst_mm, dst_pmd);
  750. src_ptl = pmd_lockptr(src_mm, src_pmd);
  751. spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
  752. ret = -EAGAIN;
  753. pmd = *src_pmd;
  754. if (unlikely(!pmd_trans_huge(pmd))) {
  755. pte_free(dst_mm, pgtable);
  756. goto out_unlock;
  757. }
  758. /*
  759. * When page table lock is held, the huge zero pmd should not be
  760. * under splitting since we don't split the page itself, only pmd to
  761. * a page table.
  762. */
  763. if (is_huge_zero_pmd(pmd)) {
  764. struct page *zero_page;
  765. bool set;
  766. /*
  767. * get_huge_zero_page() will never allocate a new page here,
  768. * since we already have a zero page to copy. It just takes a
  769. * reference.
  770. */
  771. zero_page = get_huge_zero_page();
  772. set = set_huge_zero_page(pgtable, dst_mm, vma, addr, dst_pmd,
  773. zero_page);
  774. BUG_ON(!set); /* unexpected !pmd_none(dst_pmd) */
  775. ret = 0;
  776. goto out_unlock;
  777. }
  778. if (unlikely(pmd_trans_splitting(pmd))) {
  779. /* split huge page running from under us */
  780. spin_unlock(src_ptl);
  781. spin_unlock(dst_ptl);
  782. pte_free(dst_mm, pgtable);
  783. wait_split_huge_page(vma->anon_vma, src_pmd); /* src_vma */
  784. goto out;
  785. }
  786. src_page = pmd_page(pmd);
  787. VM_BUG_ON(!PageHead(src_page));
  788. get_page(src_page);
  789. page_dup_rmap(src_page);
  790. add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
  791. pmdp_set_wrprotect(src_mm, addr, src_pmd);
  792. pmd = pmd_mkold(pmd_wrprotect(pmd));
  793. pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
  794. set_pmd_at(dst_mm, addr, dst_pmd, pmd);
  795. atomic_long_inc(&dst_mm->nr_ptes);
  796. ret = 0;
  797. out_unlock:
  798. spin_unlock(src_ptl);
  799. spin_unlock(dst_ptl);
  800. out:
  801. return ret;
  802. }
  803. void huge_pmd_set_accessed(struct mm_struct *mm,
  804. struct vm_area_struct *vma,
  805. unsigned long address,
  806. pmd_t *pmd, pmd_t orig_pmd,
  807. int dirty)
  808. {
  809. spinlock_t *ptl;
  810. pmd_t entry;
  811. unsigned long haddr;
  812. ptl = pmd_lock(mm, pmd);
  813. if (unlikely(!pmd_same(*pmd, orig_pmd)))
  814. goto unlock;
  815. entry = pmd_mkyoung(orig_pmd);
  816. haddr = address & HPAGE_PMD_MASK;
  817. if (pmdp_set_access_flags(vma, haddr, pmd, entry, dirty))
  818. update_mmu_cache_pmd(vma, address, pmd);
  819. unlock:
  820. spin_unlock(ptl);
  821. }
  822. static int do_huge_pmd_wp_zero_page_fallback(struct mm_struct *mm,
  823. struct vm_area_struct *vma, unsigned long address,
  824. pmd_t *pmd, pmd_t orig_pmd, unsigned long haddr)
  825. {
  826. spinlock_t *ptl;
  827. pgtable_t pgtable;
  828. pmd_t _pmd;
  829. struct page *page;
  830. int i, ret = 0;
  831. unsigned long mmun_start; /* For mmu_notifiers */
  832. unsigned long mmun_end; /* For mmu_notifiers */
  833. page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
  834. if (!page) {
  835. ret |= VM_FAULT_OOM;
  836. goto out;
  837. }
  838. if (mem_cgroup_newpage_charge(page, mm, GFP_KERNEL)) {
  839. put_page(page);
  840. ret |= VM_FAULT_OOM;
  841. goto out;
  842. }
  843. clear_user_highpage(page, address);
  844. __SetPageUptodate(page);
  845. mmun_start = haddr;
  846. mmun_end = haddr + HPAGE_PMD_SIZE;
  847. mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
  848. ptl = pmd_lock(mm, pmd);
  849. if (unlikely(!pmd_same(*pmd, orig_pmd)))
  850. goto out_free_page;
  851. pmdp_clear_flush(vma, haddr, pmd);
  852. /* leave pmd empty until pte is filled */
  853. pgtable = pgtable_trans_huge_withdraw(mm, pmd);
  854. pmd_populate(mm, &_pmd, pgtable);
  855. for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
  856. pte_t *pte, entry;
  857. if (haddr == (address & PAGE_MASK)) {
  858. entry = mk_pte(page, vma->vm_page_prot);
  859. entry = maybe_mkwrite(pte_mkdirty(entry), vma);
  860. page_add_new_anon_rmap(page, vma, haddr);
  861. } else {
  862. entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot);
  863. entry = pte_mkspecial(entry);
  864. }
  865. pte = pte_offset_map(&_pmd, haddr);
  866. VM_BUG_ON(!pte_none(*pte));
  867. set_pte_at(mm, haddr, pte, entry);
  868. pte_unmap(pte);
  869. }
  870. smp_wmb(); /* make pte visible before pmd */
  871. pmd_populate(mm, pmd, pgtable);
  872. spin_unlock(ptl);
  873. put_huge_zero_page();
  874. inc_mm_counter(mm, MM_ANONPAGES);
  875. mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
  876. ret |= VM_FAULT_WRITE;
  877. out:
  878. return ret;
  879. out_free_page:
  880. spin_unlock(ptl);
  881. mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
  882. mem_cgroup_uncharge_page(page);
  883. put_page(page);
  884. goto out;
  885. }
  886. static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm,
  887. struct vm_area_struct *vma,
  888. unsigned long address,
  889. pmd_t *pmd, pmd_t orig_pmd,
  890. struct page *page,
  891. unsigned long haddr)
  892. {
  893. spinlock_t *ptl;
  894. pgtable_t pgtable;
  895. pmd_t _pmd;
  896. int ret = 0, i;
  897. struct page **pages;
  898. unsigned long mmun_start; /* For mmu_notifiers */
  899. unsigned long mmun_end; /* For mmu_notifiers */
  900. pages = kmalloc(sizeof(struct page *) * HPAGE_PMD_NR,
  901. GFP_KERNEL);
  902. if (unlikely(!pages)) {
  903. ret |= VM_FAULT_OOM;
  904. goto out;
  905. }
  906. for (i = 0; i < HPAGE_PMD_NR; i++) {
  907. pages[i] = alloc_page_vma_node(GFP_HIGHUSER_MOVABLE |
  908. __GFP_OTHER_NODE,
  909. vma, address, page_to_nid(page));
  910. if (unlikely(!pages[i] ||
  911. mem_cgroup_newpage_charge(pages[i], mm,
  912. GFP_KERNEL))) {
  913. if (pages[i])
  914. put_page(pages[i]);
  915. mem_cgroup_uncharge_start();
  916. while (--i >= 0) {
  917. mem_cgroup_uncharge_page(pages[i]);
  918. put_page(pages[i]);
  919. }
  920. mem_cgroup_uncharge_end();
  921. kfree(pages);
  922. ret |= VM_FAULT_OOM;
  923. goto out;
  924. }
  925. }
  926. for (i = 0; i < HPAGE_PMD_NR; i++) {
  927. copy_user_highpage(pages[i], page + i,
  928. haddr + PAGE_SIZE * i, vma);
  929. __SetPageUptodate(pages[i]);
  930. cond_resched();
  931. }
  932. mmun_start = haddr;
  933. mmun_end = haddr + HPAGE_PMD_SIZE;
  934. mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
  935. ptl = pmd_lock(mm, pmd);
  936. if (unlikely(!pmd_same(*pmd, orig_pmd)))
  937. goto out_free_pages;
  938. VM_BUG_ON(!PageHead(page));
  939. pmdp_clear_flush(vma, haddr, pmd);
  940. /* leave pmd empty until pte is filled */
  941. pgtable = pgtable_trans_huge_withdraw(mm, pmd);
  942. pmd_populate(mm, &_pmd, pgtable);
  943. for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
  944. pte_t *pte, entry;
  945. entry = mk_pte(pages[i], vma->vm_page_prot);
  946. entry = maybe_mkwrite(pte_mkdirty(entry), vma);
  947. page_add_new_anon_rmap(pages[i], vma, haddr);
  948. pte = pte_offset_map(&_pmd, haddr);
  949. VM_BUG_ON(!pte_none(*pte));
  950. set_pte_at(mm, haddr, pte, entry);
  951. pte_unmap(pte);
  952. }
  953. kfree(pages);
  954. smp_wmb(); /* make pte visible before pmd */
  955. pmd_populate(mm, pmd, pgtable);
  956. page_remove_rmap(page);
  957. spin_unlock(ptl);
  958. mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
  959. ret |= VM_FAULT_WRITE;
  960. put_page(page);
  961. out:
  962. return ret;
  963. out_free_pages:
  964. spin_unlock(ptl);
  965. mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
  966. mem_cgroup_uncharge_start();
  967. for (i = 0; i < HPAGE_PMD_NR; i++) {
  968. mem_cgroup_uncharge_page(pages[i]);
  969. put_page(pages[i]);
  970. }
  971. mem_cgroup_uncharge_end();
  972. kfree(pages);
  973. goto out;
  974. }
  975. int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
  976. unsigned long address, pmd_t *pmd, pmd_t orig_pmd)
  977. {
  978. spinlock_t *ptl;
  979. int ret = 0;
  980. struct page *page = NULL, *new_page;
  981. unsigned long haddr;
  982. unsigned long mmun_start; /* For mmu_notifiers */
  983. unsigned long mmun_end; /* For mmu_notifiers */
  984. ptl = pmd_lockptr(mm, pmd);
  985. VM_BUG_ON(!vma->anon_vma);
  986. haddr = address & HPAGE_PMD_MASK;
  987. if (is_huge_zero_pmd(orig_pmd))
  988. goto alloc;
  989. spin_lock(ptl);
  990. if (unlikely(!pmd_same(*pmd, orig_pmd)))
  991. goto out_unlock;
  992. page = pmd_page(orig_pmd);
  993. VM_BUG_ON(!PageCompound(page) || !PageHead(page));
  994. if (page_mapcount(page) == 1) {
  995. pmd_t entry;
  996. entry = pmd_mkyoung(orig_pmd);
  997. entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
  998. if (pmdp_set_access_flags(vma, haddr, pmd, entry, 1))
  999. update_mmu_cache_pmd(vma, address, pmd);
  1000. ret |= VM_FAULT_WRITE;
  1001. goto out_unlock;
  1002. }
  1003. get_page(page);
  1004. spin_unlock(ptl);
  1005. alloc:
  1006. if (transparent_hugepage_enabled(vma) &&
  1007. !transparent_hugepage_debug_cow())
  1008. new_page = alloc_hugepage_vma(transparent_hugepage_defrag(vma),
  1009. vma, haddr, numa_node_id(), 0);
  1010. else
  1011. new_page = NULL;
  1012. if (unlikely(!new_page)) {
  1013. if (is_huge_zero_pmd(orig_pmd)) {
  1014. ret = do_huge_pmd_wp_zero_page_fallback(mm, vma,
  1015. address, pmd, orig_pmd, haddr);
  1016. } else {
  1017. ret = do_huge_pmd_wp_page_fallback(mm, vma, address,
  1018. pmd, orig_pmd, page, haddr);
  1019. if (ret & VM_FAULT_OOM)
  1020. split_huge_page(page);
  1021. put_page(page);
  1022. }
  1023. count_vm_event(THP_FAULT_FALLBACK);
  1024. goto out;
  1025. }
  1026. if (unlikely(mem_cgroup_newpage_charge(new_page, mm, GFP_KERNEL))) {
  1027. put_page(new_page);
  1028. if (page) {
  1029. split_huge_page(page);
  1030. put_page(page);
  1031. }
  1032. count_vm_event(THP_FAULT_FALLBACK);
  1033. ret |= VM_FAULT_OOM;
  1034. goto out;
  1035. }
  1036. count_vm_event(THP_FAULT_ALLOC);
  1037. if (is_huge_zero_pmd(orig_pmd))
  1038. clear_huge_page(new_page, haddr, HPAGE_PMD_NR);
  1039. else
  1040. copy_user_huge_page(new_page, page, haddr, vma, HPAGE_PMD_NR);
  1041. __SetPageUptodate(new_page);
  1042. mmun_start = haddr;
  1043. mmun_end = haddr + HPAGE_PMD_SIZE;
  1044. mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
  1045. spin_lock(ptl);
  1046. if (page)
  1047. put_page(page);
  1048. if (unlikely(!pmd_same(*pmd, orig_pmd))) {
  1049. spin_unlock(ptl);
  1050. mem_cgroup_uncharge_page(new_page);
  1051. put_page(new_page);
  1052. goto out_mn;
  1053. } else {
  1054. pmd_t entry;
  1055. entry = mk_huge_pmd(new_page, vma->vm_page_prot);
  1056. entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
  1057. pmdp_clear_flush(vma, haddr, pmd);
  1058. page_add_new_anon_rmap(new_page, vma, haddr);
  1059. set_pmd_at(mm, haddr, pmd, entry);
  1060. update_mmu_cache_pmd(vma, address, pmd);
  1061. if (is_huge_zero_pmd(orig_pmd)) {
  1062. add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR);
  1063. put_huge_zero_page();
  1064. } else {
  1065. VM_BUG_ON(!PageHead(page));
  1066. page_remove_rmap(page);
  1067. put_page(page);
  1068. }
  1069. ret |= VM_FAULT_WRITE;
  1070. }
  1071. spin_unlock(ptl);
  1072. out_mn:
  1073. mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
  1074. out:
  1075. return ret;
  1076. out_unlock:
  1077. spin_unlock(ptl);
  1078. return ret;
  1079. }
  1080. struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
  1081. unsigned long addr,
  1082. pmd_t *pmd,
  1083. unsigned int flags)
  1084. {
  1085. struct mm_struct *mm = vma->vm_mm;
  1086. struct page *page = NULL;
  1087. assert_spin_locked(pmd_lockptr(mm, pmd));
  1088. if (flags & FOLL_WRITE && !pmd_write(*pmd))
  1089. goto out;
  1090. /* Avoid dumping huge zero page */
  1091. if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd))
  1092. return ERR_PTR(-EFAULT);
  1093. page = pmd_page(*pmd);
  1094. VM_BUG_ON(!PageHead(page));
  1095. if (flags & FOLL_TOUCH) {
  1096. pmd_t _pmd;
  1097. /*
  1098. * We should set the dirty bit only for FOLL_WRITE but
  1099. * for now the dirty bit in the pmd is meaningless.
  1100. * And if the dirty bit will become meaningful and
  1101. * we'll only set it with FOLL_WRITE, an atomic
  1102. * set_bit will be required on the pmd to set the
  1103. * young bit, instead of the current set_pmd_at.
  1104. */
  1105. _pmd = pmd_mkyoung(pmd_mkdirty(*pmd));
  1106. if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK,
  1107. pmd, _pmd, 1))
  1108. update_mmu_cache_pmd(vma, addr, pmd);
  1109. }
  1110. if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
  1111. if (page->mapping && trylock_page(page)) {
  1112. lru_add_drain();
  1113. if (page->mapping)
  1114. mlock_vma_page(page);
  1115. unlock_page(page);
  1116. }
  1117. }
  1118. page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;
  1119. VM_BUG_ON(!PageCompound(page));
  1120. if (flags & FOLL_GET)
  1121. get_page_foll(page);
  1122. out:
  1123. return page;
  1124. }
  1125. /* NUMA hinting page fault entry point for trans huge pmds */
  1126. int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
  1127. unsigned long addr, pmd_t pmd, pmd_t *pmdp)
  1128. {
  1129. spinlock_t *ptl;
  1130. struct anon_vma *anon_vma = NULL;
  1131. struct page *page;
  1132. unsigned long haddr = addr & HPAGE_PMD_MASK;
  1133. int page_nid = -1, this_nid = numa_node_id();
  1134. int target_nid, last_cpupid = -1;
  1135. bool page_locked;
  1136. bool migrated = false;
  1137. int flags = 0;
  1138. ptl = pmd_lock(mm, pmdp);
  1139. if (unlikely(!pmd_same(pmd, *pmdp)))
  1140. goto out_unlock;
  1141. page = pmd_page(pmd);
  1142. BUG_ON(is_huge_zero_page(page));
  1143. page_nid = page_to_nid(page);
  1144. last_cpupid = page_cpupid_last(page);
  1145. count_vm_numa_event(NUMA_HINT_FAULTS);
  1146. if (page_nid == this_nid) {
  1147. count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
  1148. flags |= TNF_FAULT_LOCAL;
  1149. }
  1150. /*
  1151. * Avoid grouping on DSO/COW pages in specific and RO pages
  1152. * in general, RO pages shouldn't hurt as much anyway since
  1153. * they can be in shared cache state.
  1154. */
  1155. if (!pmd_write(pmd))
  1156. flags |= TNF_NO_GROUP;
  1157. /*
  1158. * Acquire the page lock to serialise THP migrations but avoid dropping
  1159. * page_table_lock if at all possible
  1160. */
  1161. page_locked = trylock_page(page);
  1162. target_nid = mpol_misplaced(page, vma, haddr);
  1163. if (target_nid == -1) {
  1164. /* If the page was locked, there are no parallel migrations */
  1165. if (page_locked)
  1166. goto clear_pmdnuma;
  1167. /*
  1168. * Otherwise wait for potential migrations and retry. We do
  1169. * relock and check_same as the page may no longer be mapped.
  1170. * As the fault is being retried, do not account for it.
  1171. */
  1172. spin_unlock(ptl);
  1173. wait_on_page_locked(page);
  1174. page_nid = -1;
  1175. goto out;
  1176. }
  1177. /* Page is misplaced, serialise migrations and parallel THP splits */
  1178. get_page(page);
  1179. spin_unlock(ptl);
  1180. if (!page_locked)
  1181. lock_page(page);
  1182. anon_vma = page_lock_anon_vma_read(page);
  1183. /* Confirm the PMD did not change while page_table_lock was released */
  1184. spin_lock(ptl);
  1185. if (unlikely(!pmd_same(pmd, *pmdp))) {
  1186. unlock_page(page);
  1187. put_page(page);
  1188. page_nid = -1;
  1189. goto out_unlock;
  1190. }
  1191. /*
  1192. * Migrate the THP to the requested node, returns with page unlocked
  1193. * and pmd_numa cleared.
  1194. */
  1195. spin_unlock(ptl);
  1196. migrated = migrate_misplaced_transhuge_page(mm, vma,
  1197. pmdp, pmd, addr, page, target_nid);
  1198. if (migrated) {
  1199. flags |= TNF_MIGRATED;
  1200. page_nid = target_nid;
  1201. }
  1202. goto out;
  1203. clear_pmdnuma:
  1204. BUG_ON(!PageLocked(page));
  1205. pmd = pmd_mknonnuma(pmd);
  1206. set_pmd_at(mm, haddr, pmdp, pmd);
  1207. VM_BUG_ON(pmd_numa(*pmdp));
  1208. update_mmu_cache_pmd(vma, addr, pmdp);
  1209. unlock_page(page);
  1210. out_unlock:
  1211. spin_unlock(ptl);
  1212. out:
  1213. if (anon_vma)
  1214. page_unlock_anon_vma_read(anon_vma);
  1215. if (page_nid != -1)
  1216. task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR, flags);
  1217. return 0;
  1218. }
  1219. int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
  1220. pmd_t *pmd, unsigned long addr)
  1221. {
  1222. spinlock_t *ptl;
  1223. int ret = 0;
  1224. if (__pmd_trans_huge_lock(pmd, vma, &ptl) == 1) {
  1225. struct page *page;
  1226. pgtable_t pgtable;
  1227. pmd_t orig_pmd;
  1228. /*
  1229. * For architectures like ppc64 we look at deposited pgtable
  1230. * when calling pmdp_get_and_clear. So do the
  1231. * pgtable_trans_huge_withdraw after finishing pmdp related
  1232. * operations.
  1233. */
  1234. orig_pmd = pmdp_get_and_clear(tlb->mm, addr, pmd);
  1235. tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
  1236. pgtable = pgtable_trans_huge_withdraw(tlb->mm, pmd);
  1237. if (is_huge_zero_pmd(orig_pmd)) {
  1238. atomic_long_dec(&tlb->mm->nr_ptes);
  1239. spin_unlock(ptl);
  1240. put_huge_zero_page();
  1241. } else {
  1242. page = pmd_page(orig_pmd);
  1243. page_remove_rmap(page);
  1244. VM_BUG_ON(page_mapcount(page) < 0);
  1245. add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
  1246. VM_BUG_ON(!PageHead(page));
  1247. atomic_long_dec(&tlb->mm->nr_ptes);
  1248. spin_unlock(ptl);
  1249. tlb_remove_page(tlb, page);
  1250. }
  1251. pte_free(tlb->mm, pgtable);
  1252. ret = 1;
  1253. }
  1254. return ret;
  1255. }
  1256. int mincore_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
  1257. unsigned long addr, unsigned long end,
  1258. unsigned char *vec)
  1259. {
  1260. spinlock_t *ptl;
  1261. int ret = 0;
  1262. if (__pmd_trans_huge_lock(pmd, vma, &ptl) == 1) {
  1263. /*
  1264. * All logical pages in the range are present
  1265. * if backed by a huge page.
  1266. */
  1267. spin_unlock(ptl);
  1268. memset(vec, 1, (end - addr) >> PAGE_SHIFT);
  1269. ret = 1;
  1270. }
  1271. return ret;
  1272. }
  1273. int move_huge_pmd(struct vm_area_struct *vma, struct vm_area_struct *new_vma,
  1274. unsigned long old_addr,
  1275. unsigned long new_addr, unsigned long old_end,
  1276. pmd_t *old_pmd, pmd_t *new_pmd)
  1277. {
  1278. spinlock_t *old_ptl, *new_ptl;
  1279. int ret = 0;
  1280. pmd_t pmd;
  1281. struct mm_struct *mm = vma->vm_mm;
  1282. if ((old_addr & ~HPAGE_PMD_MASK) ||
  1283. (new_addr & ~HPAGE_PMD_MASK) ||
  1284. old_end - old_addr < HPAGE_PMD_SIZE ||
  1285. (new_vma->vm_flags & VM_NOHUGEPAGE))
  1286. goto out;
  1287. /*
  1288. * The destination pmd shouldn't be established, free_pgtables()
  1289. * should have release it.
  1290. */
  1291. if (WARN_ON(!pmd_none(*new_pmd))) {
  1292. VM_BUG_ON(pmd_trans_huge(*new_pmd));
  1293. goto out;
  1294. }
  1295. /*
  1296. * We don't have to worry about the ordering of src and dst
  1297. * ptlocks because exclusive mmap_sem prevents deadlock.
  1298. */
  1299. ret = __pmd_trans_huge_lock(old_pmd, vma, &old_ptl);
  1300. if (ret == 1) {
  1301. new_ptl = pmd_lockptr(mm, new_pmd);
  1302. if (new_ptl != old_ptl)
  1303. spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
  1304. pmd = pmdp_get_and_clear(mm, old_addr, old_pmd);
  1305. VM_BUG_ON(!pmd_none(*new_pmd));
  1306. set_pmd_at(mm, new_addr, new_pmd, pmd_mksoft_dirty(pmd));
  1307. if (new_ptl != old_ptl)
  1308. spin_unlock(new_ptl);
  1309. spin_unlock(old_ptl);
  1310. }
  1311. out:
  1312. return ret;
  1313. }
  1314. /*
  1315. * Returns
  1316. * - 0 if PMD could not be locked
  1317. * - 1 if PMD was locked but protections unchange and TLB flush unnecessary
  1318. * - HPAGE_PMD_NR is protections changed and TLB flush necessary
  1319. */
  1320. int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
  1321. unsigned long addr, pgprot_t newprot, int prot_numa)
  1322. {
  1323. struct mm_struct *mm = vma->vm_mm;
  1324. spinlock_t *ptl;
  1325. int ret = 0;
  1326. if (__pmd_trans_huge_lock(pmd, vma, &ptl) == 1) {
  1327. pmd_t entry;
  1328. ret = 1;
  1329. if (!prot_numa) {
  1330. entry = pmdp_get_and_clear(mm, addr, pmd);
  1331. entry = pmd_modify(entry, newprot);
  1332. ret = HPAGE_PMD_NR;
  1333. BUG_ON(pmd_write(entry));
  1334. } else {
  1335. struct page *page = pmd_page(*pmd);
  1336. /*
  1337. * Do not trap faults against the zero page. The
  1338. * read-only data is likely to be read-cached on the
  1339. * local CPU cache and it is less useful to know about
  1340. * local vs remote hits on the zero page.
  1341. */
  1342. if (!is_huge_zero_page(page) &&
  1343. !pmd_numa(*pmd)) {
  1344. entry = pmdp_get_and_clear(mm, addr, pmd);
  1345. entry = pmd_mknuma(entry);
  1346. ret = HPAGE_PMD_NR;
  1347. }
  1348. }
  1349. /* Set PMD if cleared earlier */
  1350. if (ret == HPAGE_PMD_NR)
  1351. set_pmd_at(mm, addr, pmd, entry);
  1352. spin_unlock(ptl);
  1353. }
  1354. return ret;
  1355. }
  1356. /*
  1357. * Returns 1 if a given pmd maps a stable (not under splitting) thp.
  1358. * Returns -1 if it maps a thp under splitting. Returns 0 otherwise.
  1359. *
  1360. * Note that if it returns 1, this routine returns without unlocking page
  1361. * table locks. So callers must unlock them.
  1362. */
  1363. int __pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma,
  1364. spinlock_t **ptl)
  1365. {
  1366. *ptl = pmd_lock(vma->vm_mm, pmd);
  1367. if (likely(pmd_trans_huge(*pmd))) {
  1368. if (unlikely(pmd_trans_splitting(*pmd))) {
  1369. spin_unlock(*ptl);
  1370. wait_split_huge_page(vma->anon_vma, pmd);
  1371. return -1;
  1372. } else {
  1373. /* Thp mapped by 'pmd' is stable, so we can
  1374. * handle it as it is. */
  1375. return 1;
  1376. }
  1377. }
  1378. spin_unlock(*ptl);
  1379. return 0;
  1380. }
  1381. /*
  1382. * This function returns whether a given @page is mapped onto the @address
  1383. * in the virtual space of @mm.
  1384. *
  1385. * When it's true, this function returns *pmd with holding the page table lock
  1386. * and passing it back to the caller via @ptl.
  1387. * If it's false, returns NULL without holding the page table lock.
  1388. */
  1389. pmd_t *page_check_address_pmd(struct page *page,
  1390. struct mm_struct *mm,
  1391. unsigned long address,
  1392. enum page_check_address_pmd_flag flag,
  1393. spinlock_t **ptl)
  1394. {
  1395. pmd_t *pmd;
  1396. if (address & ~HPAGE_PMD_MASK)
  1397. return NULL;
  1398. pmd = mm_find_pmd(mm, address);
  1399. if (!pmd)
  1400. return NULL;
  1401. *ptl = pmd_lock(mm, pmd);
  1402. if (pmd_none(*pmd))
  1403. goto unlock;
  1404. if (pmd_page(*pmd) != page)
  1405. goto unlock;
  1406. /*
  1407. * split_vma() may create temporary aliased mappings. There is
  1408. * no risk as long as all huge pmd are found and have their
  1409. * splitting bit set before __split_huge_page_refcount
  1410. * runs. Finding the same huge pmd more than once during the
  1411. * same rmap walk is not a problem.
  1412. */
  1413. if (flag == PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG &&
  1414. pmd_trans_splitting(*pmd))
  1415. goto unlock;
  1416. if (pmd_trans_huge(*pmd)) {
  1417. VM_BUG_ON(flag == PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG &&
  1418. !pmd_trans_splitting(*pmd));
  1419. return pmd;
  1420. }
  1421. unlock:
  1422. spin_unlock(*ptl);
  1423. return NULL;
  1424. }
  1425. static int __split_huge_page_splitting(struct page *page,
  1426. struct vm_area_struct *vma,
  1427. unsigned long address)
  1428. {
  1429. struct mm_struct *mm = vma->vm_mm;
  1430. spinlock_t *ptl;
  1431. pmd_t *pmd;
  1432. int ret = 0;
  1433. /* For mmu_notifiers */
  1434. const unsigned long mmun_start = address;
  1435. const unsigned long mmun_end = address + HPAGE_PMD_SIZE;
  1436. mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
  1437. pmd = page_check_address_pmd(page, mm, address,
  1438. PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG, &ptl);
  1439. if (pmd) {
  1440. /*
  1441. * We can't temporarily set the pmd to null in order
  1442. * to split it, the pmd must remain marked huge at all
  1443. * times or the VM won't take the pmd_trans_huge paths
  1444. * and it won't wait on the anon_vma->root->rwsem to
  1445. * serialize against split_huge_page*.
  1446. */
  1447. pmdp_splitting_flush(vma, address, pmd);
  1448. ret = 1;
  1449. spin_unlock(ptl);
  1450. }
  1451. mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
  1452. return ret;
  1453. }
  1454. static void __split_huge_page_refcount(struct page *page,
  1455. struct list_head *list)
  1456. {
  1457. int i;
  1458. struct zone *zone = page_zone(page);
  1459. struct lruvec *lruvec;
  1460. int tail_count = 0;
  1461. /* prevent PageLRU to go away from under us, and freeze lru stats */
  1462. spin_lock_irq(&zone->lru_lock);
  1463. lruvec = mem_cgroup_page_lruvec(page, zone);
  1464. compound_lock(page);
  1465. /* complete memcg works before add pages to LRU */
  1466. mem_cgroup_split_huge_fixup(page);
  1467. for (i = HPAGE_PMD_NR - 1; i >= 1; i--) {
  1468. struct page *page_tail = page + i;
  1469. /* tail_page->_mapcount cannot change */
  1470. BUG_ON(page_mapcount(page_tail) < 0);
  1471. tail_count += page_mapcount(page_tail);
  1472. /* check for overflow */
  1473. BUG_ON(tail_count < 0);
  1474. BUG_ON(atomic_read(&page_tail->_count) != 0);
  1475. /*
  1476. * tail_page->_count is zero and not changing from
  1477. * under us. But get_page_unless_zero() may be running
  1478. * from under us on the tail_page. If we used
  1479. * atomic_set() below instead of atomic_add(), we
  1480. * would then run atomic_set() concurrently with
  1481. * get_page_unless_zero(), and atomic_set() is
  1482. * implemented in C not using locked ops. spin_unlock
  1483. * on x86 sometime uses locked ops because of PPro
  1484. * errata 66, 92, so unless somebody can guarantee
  1485. * atomic_set() here would be safe on all archs (and
  1486. * not only on x86), it's safer to use atomic_add().
  1487. */
  1488. atomic_add(page_mapcount(page) + page_mapcount(page_tail) + 1,
  1489. &page_tail->_count);
  1490. /* after clearing PageTail the gup refcount can be released */
  1491. smp_mb();
  1492. /*
  1493. * retain hwpoison flag of the poisoned tail page:
  1494. * fix for the unsuitable process killed on Guest Machine(KVM)
  1495. * by the memory-failure.
  1496. */
  1497. page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP | __PG_HWPOISON;
  1498. page_tail->flags |= (page->flags &
  1499. ((1L << PG_referenced) |
  1500. (1L << PG_swapbacked) |
  1501. (1L << PG_mlocked) |
  1502. (1L << PG_uptodate) |
  1503. (1L << PG_active) |
  1504. (1L << PG_unevictable)));
  1505. page_tail->flags |= (1L << PG_dirty);
  1506. /* clear PageTail before overwriting first_page */
  1507. smp_wmb();
  1508. /*
  1509. * __split_huge_page_splitting() already set the
  1510. * splitting bit in all pmd that could map this
  1511. * hugepage, that will ensure no CPU can alter the
  1512. * mapcount on the head page. The mapcount is only
  1513. * accounted in the head page and it has to be
  1514. * transferred to all tail pages in the below code. So
  1515. * for this code to be safe, the split the mapcount
  1516. * can't change. But that doesn't mean userland can't
  1517. * keep changing and reading the page contents while
  1518. * we transfer the mapcount, so the pmd splitting
  1519. * status is achieved setting a reserved bit in the
  1520. * pmd, not by clearing the present bit.
  1521. */
  1522. page_tail->_mapcount = page->_mapcount;
  1523. BUG_ON(page_tail->mapping);
  1524. page_tail->mapping = page->mapping;
  1525. page_tail->index = page->index + i;
  1526. page_cpupid_xchg_last(page_tail, page_cpupid_last(page));
  1527. BUG_ON(!PageAnon(page_tail));
  1528. BUG_ON(!PageUptodate(page_tail));
  1529. BUG_ON(!PageDirty(page_tail));
  1530. BUG_ON(!PageSwapBacked(page_tail));
  1531. lru_add_page_tail(page, page_tail, lruvec, list);
  1532. }
  1533. atomic_sub(tail_count, &page->_count);
  1534. BUG_ON(atomic_read(&page->_count) <= 0);
  1535. __mod_zone_page_state(zone, NR_ANON_TRANSPARENT_HUGEPAGES, -1);
  1536. ClearPageCompound(page);
  1537. compound_unlock(page);
  1538. spin_unlock_irq(&zone->lru_lock);
  1539. for (i = 1; i < HPAGE_PMD_NR; i++) {
  1540. struct page *page_tail = page + i;
  1541. BUG_ON(page_count(page_tail) <= 0);
  1542. /*
  1543. * Tail pages may be freed if there wasn't any mapping
  1544. * like if add_to_swap() is running on a lru page that
  1545. * had its mapping zapped. And freeing these pages
  1546. * requires taking the lru_lock so we do the put_page
  1547. * of the tail pages after the split is complete.
  1548. */
  1549. put_page(page_tail);
  1550. }
  1551. /*
  1552. * Only the head page (now become a regular page) is required
  1553. * to be pinned by the caller.
  1554. */
  1555. BUG_ON(page_count(page) <= 0);
  1556. }
  1557. static int __split_huge_page_map(struct page *page,
  1558. struct vm_area_struct *vma,
  1559. unsigned long address)
  1560. {
  1561. struct mm_struct *mm = vma->vm_mm;
  1562. spinlock_t *ptl;
  1563. pmd_t *pmd, _pmd;
  1564. int ret = 0, i;
  1565. pgtable_t pgtable;
  1566. unsigned long haddr;
  1567. pmd = page_check_address_pmd(page, mm, address,
  1568. PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG, &ptl);
  1569. if (pmd) {
  1570. pgtable = pgtable_trans_huge_withdraw(mm, pmd);
  1571. pmd_populate(mm, &_pmd, pgtable);
  1572. haddr = address;
  1573. for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
  1574. pte_t *pte, entry;
  1575. BUG_ON(PageCompound(page+i));
  1576. entry = mk_pte(page + i, vma->vm_page_prot);
  1577. entry = maybe_mkwrite(pte_mkdirty(entry), vma);
  1578. if (!pmd_write(*pmd))
  1579. entry = pte_wrprotect(entry);
  1580. else
  1581. BUG_ON(page_mapcount(page) != 1);
  1582. if (!pmd_young(*pmd))
  1583. entry = pte_mkold(entry);
  1584. if (pmd_numa(*pmd))
  1585. entry = pte_mknuma(entry);
  1586. pte = pte_offset_map(&_pmd, haddr);
  1587. BUG_ON(!pte_none(*pte));
  1588. set_pte_at(mm, haddr, pte, entry);
  1589. pte_unmap(pte);
  1590. }
  1591. smp_wmb(); /* make pte visible before pmd */
  1592. /*
  1593. * Up to this point the pmd is present and huge and
  1594. * userland has the whole access to the hugepage
  1595. * during the split (which happens in place). If we
  1596. * overwrite the pmd with the not-huge version
  1597. * pointing to the pte here (which of course we could
  1598. * if all CPUs were bug free), userland could trigger
  1599. * a small page size TLB miss on the small sized TLB
  1600. * while the hugepage TLB entry is still established
  1601. * in the huge TLB. Some CPU doesn't like that. See
  1602. * http://support.amd.com/us/Processor_TechDocs/41322.pdf,
  1603. * Erratum 383 on page 93. Intel should be safe but is
  1604. * also warns that it's only safe if the permission
  1605. * and cache attributes of the two entries loaded in
  1606. * the two TLB is identical (which should be the case
  1607. * here). But it is generally safer to never allow
  1608. * small and huge TLB entries for the same virtual
  1609. * address to be loaded simultaneously. So instead of
  1610. * doing "pmd_populate(); flush_tlb_range();" we first
  1611. * mark the current pmd notpresent (atomically because
  1612. * here the pmd_trans_huge and pmd_trans_splitting
  1613. * must remain set at all times on the pmd until the
  1614. * split is complete for this pmd), then we flush the
  1615. * SMP TLB and finally we write the non-huge version
  1616. * of the pmd entry with pmd_populate.
  1617. */
  1618. pmdp_invalidate(vma, address, pmd);
  1619. pmd_populate(mm, pmd, pgtable);
  1620. ret = 1;
  1621. spin_unlock(ptl);
  1622. }
  1623. return ret;
  1624. }
  1625. /* must be called with anon_vma->root->rwsem held */
  1626. static void __split_huge_page(struct page *page,
  1627. struct anon_vma *anon_vma,
  1628. struct list_head *list)
  1629. {
  1630. int mapcount, mapcount2;
  1631. pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
  1632. struct anon_vma_chain *avc;
  1633. BUG_ON(!PageHead(page));
  1634. BUG_ON(PageTail(page));
  1635. mapcount = 0;
  1636. anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff, pgoff) {
  1637. struct vm_area_struct *vma = avc->vma;
  1638. unsigned long addr = vma_address(page, vma);
  1639. BUG_ON(is_vma_temporary_stack(vma));
  1640. mapcount += __split_huge_page_splitting(page, vma, addr);
  1641. }
  1642. /*
  1643. * It is critical that new vmas are added to the tail of the
  1644. * anon_vma list. This guarantes that if copy_huge_pmd() runs
  1645. * and establishes a child pmd before
  1646. * __split_huge_page_splitting() freezes the parent pmd (so if
  1647. * we fail to prevent copy_huge_pmd() from running until the
  1648. * whole __split_huge_page() is complete), we will still see
  1649. * the newly established pmd of the child later during the
  1650. * walk, to be able to set it as pmd_trans_splitting too.
  1651. */
  1652. if (mapcount != page_mapcount(page))
  1653. printk(KERN_ERR "mapcount %d page_mapcount %d\n",
  1654. mapcount, page_mapcount(page));
  1655. BUG_ON(mapcount != page_mapcount(page));
  1656. __split_huge_page_refcount(page, list);
  1657. mapcount2 = 0;
  1658. anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff, pgoff) {
  1659. struct vm_area_struct *vma = avc->vma;
  1660. unsigned long addr = vma_address(page, vma);
  1661. BUG_ON(is_vma_temporary_stack(vma));
  1662. mapcount2 += __split_huge_page_map(page, vma, addr);
  1663. }
  1664. if (mapcount != mapcount2)
  1665. printk(KERN_ERR "mapcount %d mapcount2 %d page_mapcount %d\n",
  1666. mapcount, mapcount2, page_mapcount(page));
  1667. BUG_ON(mapcount != mapcount2);
  1668. }
  1669. /*
  1670. * Split a hugepage into normal pages. This doesn't change the position of head
  1671. * page. If @list is null, tail pages will be added to LRU list, otherwise, to
  1672. * @list. Both head page and tail pages will inherit mapping, flags, and so on
  1673. * from the hugepage.
  1674. * Return 0 if the hugepage is split successfully otherwise return 1.
  1675. */
  1676. int split_huge_page_to_list(struct page *page, struct list_head *list)
  1677. {
  1678. struct anon_vma *anon_vma;
  1679. int ret = 1;
  1680. BUG_ON(is_huge_zero_page(page));
  1681. BUG_ON(!PageAnon(page));
  1682. /*
  1683. * The caller does not necessarily hold an mmap_sem that would prevent
  1684. * the anon_vma disappearing so we first we take a reference to it
  1685. * and then lock the anon_vma for write. This is similar to
  1686. * page_lock_anon_vma_read except the write lock is taken to serialise
  1687. * against parallel split or collapse operations.
  1688. */
  1689. anon_vma = page_get_anon_vma(page);
  1690. if (!anon_vma)
  1691. goto out;
  1692. anon_vma_lock_write(anon_vma);
  1693. ret = 0;
  1694. if (!PageCompound(page))
  1695. goto out_unlock;
  1696. BUG_ON(!PageSwapBacked(page));
  1697. __split_huge_page(page, anon_vma, list);
  1698. count_vm_event(THP_SPLIT);
  1699. BUG_ON(PageCompound(page));
  1700. out_unlock:
  1701. anon_vma_unlock_write(anon_vma);
  1702. put_anon_vma(anon_vma);
  1703. out:
  1704. return ret;
  1705. }
  1706. #define VM_NO_THP (VM_SPECIAL|VM_MIXEDMAP|VM_HUGETLB|VM_SHARED|VM_MAYSHARE)
  1707. int hugepage_madvise(struct vm_area_struct *vma,
  1708. unsigned long *vm_flags, int advice)
  1709. {
  1710. struct mm_struct *mm = vma->vm_mm;
  1711. switch (advice) {
  1712. case MADV_HUGEPAGE:
  1713. /*
  1714. * Be somewhat over-protective like KSM for now!
  1715. */
  1716. if (*vm_flags & (VM_HUGEPAGE | VM_NO_THP))
  1717. return -EINVAL;
  1718. if (mm->def_flags & VM_NOHUGEPAGE)
  1719. return -EINVAL;
  1720. *vm_flags &= ~VM_NOHUGEPAGE;
  1721. *vm_flags |= VM_HUGEPAGE;
  1722. /*
  1723. * If the vma become good for khugepaged to scan,
  1724. * register it here without waiting a page fault that
  1725. * may not happen any time soon.
  1726. */
  1727. if (unlikely(khugepaged_enter_vma_merge(vma)))
  1728. return -ENOMEM;
  1729. break;
  1730. case MADV_NOHUGEPAGE:
  1731. /*
  1732. * Be somewhat over-protective like KSM for now!
  1733. */
  1734. if (*vm_flags & (VM_NOHUGEPAGE | VM_NO_THP))
  1735. return -EINVAL;
  1736. *vm_flags &= ~VM_HUGEPAGE;
  1737. *vm_flags |= VM_NOHUGEPAGE;
  1738. /*
  1739. * Setting VM_NOHUGEPAGE will prevent khugepaged from scanning
  1740. * this vma even if we leave the mm registered in khugepaged if
  1741. * it got registered before VM_NOHUGEPAGE was set.
  1742. */
  1743. break;
  1744. }
  1745. return 0;
  1746. }
  1747. static int __init khugepaged_slab_init(void)
  1748. {
  1749. mm_slot_cache = kmem_cache_create("khugepaged_mm_slot",
  1750. sizeof(struct mm_slot),
  1751. __alignof__(struct mm_slot), 0, NULL);
  1752. if (!mm_slot_cache)
  1753. return -ENOMEM;
  1754. return 0;
  1755. }
  1756. static inline struct mm_slot *alloc_mm_slot(void)
  1757. {
  1758. if (!mm_slot_cache) /* initialization failed */
  1759. return NULL;
  1760. return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL);
  1761. }
  1762. static inline void free_mm_slot(struct mm_slot *mm_slot)
  1763. {
  1764. kmem_cache_free(mm_slot_cache, mm_slot);
  1765. }
  1766. static struct mm_slot *get_mm_slot(struct mm_struct *mm)
  1767. {
  1768. struct mm_slot *mm_slot;
  1769. hash_for_each_possible(mm_slots_hash, mm_slot, hash, (unsigned long)mm)
  1770. if (mm == mm_slot->mm)
  1771. return mm_slot;
  1772. return NULL;
  1773. }
  1774. static void insert_to_mm_slots_hash(struct mm_struct *mm,
  1775. struct mm_slot *mm_slot)
  1776. {
  1777. mm_slot->mm = mm;
  1778. hash_add(mm_slots_hash, &mm_slot->hash, (long)mm);
  1779. }
  1780. static inline int khugepaged_test_exit(struct mm_struct *mm)
  1781. {
  1782. return atomic_read(&mm->mm_users) == 0;
  1783. }
  1784. int __khugepaged_enter(struct mm_struct *mm)
  1785. {
  1786. struct mm_slot *mm_slot;
  1787. int wakeup;
  1788. mm_slot = alloc_mm_slot();
  1789. if (!mm_slot)
  1790. return -ENOMEM;
  1791. /* __khugepaged_exit() must not run from under us */
  1792. VM_BUG_ON(khugepaged_test_exit(mm));
  1793. if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) {
  1794. free_mm_slot(mm_slot);
  1795. return 0;
  1796. }
  1797. spin_lock(&khugepaged_mm_lock);
  1798. insert_to_mm_slots_hash(mm, mm_slot);
  1799. /*
  1800. * Insert just behind the scanning cursor, to let the area settle
  1801. * down a little.
  1802. */
  1803. wakeup = list_empty(&khugepaged_scan.mm_head);
  1804. list_add_tail(&mm_slot->mm_node, &khugepaged_scan.mm_head);
  1805. spin_unlock(&khugepaged_mm_lock);
  1806. atomic_inc(&mm->mm_count);
  1807. if (wakeup)
  1808. wake_up_interruptible(&khugepaged_wait);
  1809. return 0;
  1810. }
  1811. int khugepaged_enter_vma_merge(struct vm_area_struct *vma)
  1812. {
  1813. unsigned long hstart, hend;
  1814. if (!vma->anon_vma)
  1815. /*
  1816. * Not yet faulted in so we will register later in the
  1817. * page fault if needed.
  1818. */
  1819. return 0;
  1820. if (vma->vm_ops)
  1821. /* khugepaged not yet working on file or special mappings */
  1822. return 0;
  1823. VM_BUG_ON(vma->vm_flags & VM_NO_THP);
  1824. hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
  1825. hend = vma->vm_end & HPAGE_PMD_MASK;
  1826. if (hstart < hend)
  1827. return khugepaged_enter(vma);
  1828. return 0;
  1829. }
  1830. void __khugepaged_exit(struct mm_struct *mm)
  1831. {
  1832. struct mm_slot *mm_slot;
  1833. int free = 0;
  1834. spin_lock(&khugepaged_mm_lock);
  1835. mm_slot = get_mm_slot(mm);
  1836. if (mm_slot && khugepaged_scan.mm_slot != mm_slot) {
  1837. hash_del(&mm_slot->hash);
  1838. list_del(&mm_slot->mm_node);
  1839. free = 1;
  1840. }
  1841. spin_unlock(&khugepaged_mm_lock);
  1842. if (free) {
  1843. clear_bit(MMF_VM_HUGEPAGE, &mm->flags);
  1844. free_mm_slot(mm_slot);
  1845. mmdrop(mm);
  1846. } else if (mm_slot) {
  1847. /*
  1848. * This is required to serialize against
  1849. * khugepaged_test_exit() (which is guaranteed to run
  1850. * under mmap sem read mode). Stop here (after we
  1851. * return all pagetables will be destroyed) until
  1852. * khugepaged has finished working on the pagetables
  1853. * under the mmap_sem.
  1854. */
  1855. down_write(&mm->mmap_sem);
  1856. up_write(&mm->mmap_sem);
  1857. }
  1858. }
  1859. static void release_pte_page(struct page *page)
  1860. {
  1861. /* 0 stands for page_is_file_cache(page) == false */
  1862. dec_zone_page_state(page, NR_ISOLATED_ANON + 0);
  1863. unlock_page(page);
  1864. putback_lru_page(page);
  1865. }
  1866. static void release_pte_pages(pte_t *pte, pte_t *_pte)
  1867. {
  1868. while (--_pte >= pte) {
  1869. pte_t pteval = *_pte;
  1870. if (!pte_none(pteval))
  1871. release_pte_page(pte_page(pteval));
  1872. }
  1873. }
  1874. static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
  1875. unsigned long address,
  1876. pte_t *pte)
  1877. {
  1878. struct page *page;
  1879. pte_t *_pte;
  1880. int referenced = 0, none = 0;
  1881. for (_pte = pte; _pte < pte+HPAGE_PMD_NR;
  1882. _pte++, address += PAGE_SIZE) {
  1883. pte_t pteval = *_pte;
  1884. if (pte_none(pteval)) {
  1885. if (++none <= khugepaged_max_ptes_none)
  1886. continue;
  1887. else
  1888. goto out;
  1889. }
  1890. if (!pte_present(pteval) || !pte_write(pteval))
  1891. goto out;
  1892. page = vm_normal_page(vma, address, pteval);
  1893. if (unlikely(!page))
  1894. goto out;
  1895. VM_BUG_ON(PageCompound(page));
  1896. BUG_ON(!PageAnon(page));
  1897. VM_BUG_ON(!PageSwapBacked(page));
  1898. /* cannot use mapcount: can't collapse if there's a gup pin */
  1899. if (page_count(page) != 1)
  1900. goto out;
  1901. /*
  1902. * We can do it before isolate_lru_page because the
  1903. * page can't be freed from under us. NOTE: PG_lock
  1904. * is needed to serialize against split_huge_page
  1905. * when invoked from the VM.
  1906. */
  1907. if (!trylock_page(page))
  1908. goto out;
  1909. /*
  1910. * Isolate the page to avoid collapsing an hugepage
  1911. * currently in use by the VM.
  1912. */
  1913. if (isolate_lru_page(page)) {
  1914. unlock_page(page);
  1915. goto out;
  1916. }
  1917. /* 0 stands for page_is_file_cache(page) == false */
  1918. inc_zone_page_state(page, NR_ISOLATED_ANON + 0);
  1919. VM_BUG_ON(!PageLocked(page));
  1920. VM_BUG_ON(PageLRU(page));
  1921. /* If there is no mapped pte young don't collapse the page */
  1922. if (pte_young(pteval) || PageReferenced(page) ||
  1923. mmu_notifier_test_young(vma->vm_mm, address))
  1924. referenced = 1;
  1925. }
  1926. if (likely(referenced))
  1927. return 1;
  1928. out:
  1929. release_pte_pages(pte, _pte);
  1930. return 0;
  1931. }
  1932. static void __collapse_huge_page_copy(pte_t *pte, struct page *page,
  1933. struct vm_area_struct *vma,
  1934. unsigned long address,
  1935. spinlock_t *ptl)
  1936. {
  1937. pte_t *_pte;
  1938. for (_pte = pte; _pte < pte+HPAGE_PMD_NR; _pte++) {
  1939. pte_t pteval = *_pte;
  1940. struct page *src_page;
  1941. if (pte_none(pteval)) {
  1942. clear_user_highpage(page, address);
  1943. add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1);
  1944. } else {
  1945. src_page = pte_page(pteval);
  1946. copy_user_highpage(page, src_page, address, vma);
  1947. VM_BUG_ON(page_mapcount(src_page) != 1);
  1948. release_pte_page(src_page);
  1949. /*
  1950. * ptl mostly unnecessary, but preempt has to
  1951. * be disabled to update the per-cpu stats
  1952. * inside page_remove_rmap().
  1953. */
  1954. spin_lock(ptl);
  1955. /*
  1956. * paravirt calls inside pte_clear here are
  1957. * superfluous.
  1958. */
  1959. pte_clear(vma->vm_mm, address, _pte);
  1960. page_remove_rmap(src_page);
  1961. spin_unlock(ptl);
  1962. free_page_and_swap_cache(src_page);
  1963. }
  1964. address += PAGE_SIZE;
  1965. page++;
  1966. }
  1967. }
  1968. static void khugepaged_alloc_sleep(void)
  1969. {
  1970. wait_event_freezable_timeout(khugepaged_wait, false,
  1971. msecs_to_jiffies(khugepaged_alloc_sleep_millisecs));
  1972. }
  1973. static int khugepaged_node_load[MAX_NUMNODES];
  1974. #ifdef CONFIG_NUMA
  1975. static int khugepaged_find_target_node(void)
  1976. {
  1977. static int last_khugepaged_target_node = NUMA_NO_NODE;
  1978. int nid, target_node = 0, max_value = 0;
  1979. /* find first node with max normal pages hit */
  1980. for (nid = 0; nid < MAX_NUMNODES; nid++)
  1981. if (khugepaged_node_load[nid] > max_value) {
  1982. max_value = khugepaged_node_load[nid];
  1983. target_node = nid;
  1984. }
  1985. /* do some balance if several nodes have the same hit record */
  1986. if (target_node <= last_khugepaged_target_node)
  1987. for (nid = last_khugepaged_target_node + 1; nid < MAX_NUMNODES;
  1988. nid++)
  1989. if (max_value == khugepaged_node_load[nid]) {
  1990. target_node = nid;
  1991. break;
  1992. }
  1993. last_khugepaged_target_node = target_node;
  1994. return target_node;
  1995. }
  1996. static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)
  1997. {
  1998. if (IS_ERR(*hpage)) {
  1999. if (!*wait)
  2000. return false;
  2001. *wait = false;
  2002. *hpage = NULL;
  2003. khugepaged_alloc_sleep();
  2004. } else if (*hpage) {
  2005. put_page(*hpage);
  2006. *hpage = NULL;
  2007. }
  2008. return true;
  2009. }
  2010. static struct page
  2011. *khugepaged_alloc_page(struct page **hpage, struct mm_struct *mm,
  2012. struct vm_area_struct *vma, unsigned long address,
  2013. int node)
  2014. {
  2015. VM_BUG_ON(*hpage);
  2016. /*
  2017. * Allocate the page while the vma is still valid and under
  2018. * the mmap_sem read mode so there is no memory allocation
  2019. * later when we take the mmap_sem in write mode. This is more
  2020. * friendly behavior (OTOH it may actually hide bugs) to
  2021. * filesystems in userland with daemons allocating memory in
  2022. * the userland I/O paths. Allocating memory with the
  2023. * mmap_sem in read mode is good idea also to allow greater
  2024. * scalability.
  2025. */
  2026. *hpage = alloc_pages_exact_node(node, alloc_hugepage_gfpmask(
  2027. khugepaged_defrag(), __GFP_OTHER_NODE), HPAGE_PMD_ORDER);
  2028. /*
  2029. * After allocating the hugepage, release the mmap_sem read lock in
  2030. * preparation for taking it in write mode.
  2031. */
  2032. up_read(&mm->mmap_sem);
  2033. if (unlikely(!*hpage)) {
  2034. count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
  2035. *hpage = ERR_PTR(-ENOMEM);
  2036. return NULL;
  2037. }
  2038. count_vm_event(THP_COLLAPSE_ALLOC);
  2039. return *hpage;
  2040. }
  2041. #else
  2042. static int khugepaged_find_target_node(void)
  2043. {
  2044. return 0;
  2045. }
  2046. static inline struct page *alloc_hugepage(int defrag)
  2047. {
  2048. return alloc_pages(alloc_hugepage_gfpmask(defrag, 0),
  2049. HPAGE_PMD_ORDER);
  2050. }
  2051. static struct page *khugepaged_alloc_hugepage(bool *wait)
  2052. {
  2053. struct page *hpage;
  2054. do {
  2055. hpage = alloc_hugepage(khugepaged_defrag());
  2056. if (!hpage) {
  2057. count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
  2058. if (!*wait)
  2059. return NULL;
  2060. *wait = false;
  2061. khugepaged_alloc_sleep();
  2062. } else
  2063. count_vm_event(THP_COLLAPSE_ALLOC);
  2064. } while (unlikely(!hpage) && likely(khugepaged_enabled()));
  2065. return hpage;
  2066. }
  2067. static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)
  2068. {
  2069. if (!*hpage)
  2070. *hpage = khugepaged_alloc_hugepage(wait);
  2071. if (unlikely(!*hpage))
  2072. return false;
  2073. return true;
  2074. }
  2075. static struct page
  2076. *khugepaged_alloc_page(struct page **hpage, struct mm_struct *mm,
  2077. struct vm_area_struct *vma, unsigned long address,
  2078. int node)
  2079. {
  2080. up_read(&mm->mmap_sem);
  2081. VM_BUG_ON(!*hpage);
  2082. return *hpage;
  2083. }
  2084. #endif
  2085. static bool hugepage_vma_check(struct vm_area_struct *vma)
  2086. {
  2087. if ((!(vma->vm_flags & VM_HUGEPAGE) && !khugepaged_always()) ||
  2088. (vma->vm_flags & VM_NOHUGEPAGE))
  2089. return false;
  2090. if (!vma->anon_vma || vma->vm_ops)
  2091. return false;
  2092. if (is_vma_temporary_stack(vma))
  2093. return false;
  2094. VM_BUG_ON(vma->vm_flags & VM_NO_THP);
  2095. return true;
  2096. }
  2097. static void collapse_huge_page(struct mm_struct *mm,
  2098. unsigned long address,
  2099. struct page **hpage,
  2100. struct vm_area_struct *vma,
  2101. int node)
  2102. {
  2103. pmd_t *pmd, _pmd;
  2104. pte_t *pte;
  2105. pgtable_t pgtable;
  2106. struct page *new_page;
  2107. spinlock_t *pmd_ptl, *pte_ptl;
  2108. int isolated;
  2109. unsigned long hstart, hend;
  2110. unsigned long mmun_start; /* For mmu_notifiers */
  2111. unsigned long mmun_end; /* For mmu_notifiers */
  2112. VM_BUG_ON(address & ~HPAGE_PMD_MASK);
  2113. /* release the mmap_sem read lock. */
  2114. new_page = khugepaged_alloc_page(hpage, mm, vma, address, node);
  2115. if (!new_page)
  2116. return;
  2117. if (unlikely(mem_cgroup_newpage_charge(new_page, mm, GFP_KERNEL)))
  2118. return;
  2119. /*
  2120. * Prevent all access to pagetables with the exception of
  2121. * gup_fast later hanlded by the ptep_clear_flush and the VM
  2122. * handled by the anon_vma lock + PG_lock.
  2123. */
  2124. down_write(&mm->mmap_sem);
  2125. if (unlikely(khugepaged_test_exit(mm)))
  2126. goto out;
  2127. vma = find_vma(mm, address);
  2128. if (!vma)
  2129. goto out;
  2130. hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
  2131. hend = vma->vm_end & HPAGE_PMD_MASK;
  2132. if (address < hstart || address + HPAGE_PMD_SIZE > hend)
  2133. goto out;
  2134. if (!hugepage_vma_check(vma))
  2135. goto out;
  2136. pmd = mm_find_pmd(mm, address);
  2137. if (!pmd)
  2138. goto out;
  2139. if (pmd_trans_huge(*pmd))
  2140. goto out;
  2141. anon_vma_lock_write(vma->anon_vma);
  2142. pte = pte_offset_map(pmd, address);
  2143. pte_ptl = pte_lockptr(mm, pmd);
  2144. mmun_start = address;
  2145. mmun_end = address + HPAGE_PMD_SIZE;
  2146. mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
  2147. pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */
  2148. /*
  2149. * After this gup_fast can't run anymore. This also removes
  2150. * any huge TLB entry from the CPU so we won't allow
  2151. * huge and small TLB entries for the same virtual address
  2152. * to avoid the risk of CPU bugs in that area.
  2153. */
  2154. _pmd = pmdp_clear_flush(vma, address, pmd);
  2155. spin_unlock(pmd_ptl);
  2156. mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
  2157. spin_lock(pte_ptl);
  2158. isolated = __collapse_huge_page_isolate(vma, address, pte);
  2159. spin_unlock(pte_ptl);
  2160. if (unlikely(!isolated)) {
  2161. pte_unmap(pte);
  2162. spin_lock(pmd_ptl);
  2163. BUG_ON(!pmd_none(*pmd));
  2164. /*
  2165. * We can only use set_pmd_at when establishing
  2166. * hugepmds and never for establishing regular pmds that
  2167. * points to regular pagetables. Use pmd_populate for that
  2168. */
  2169. pmd_populate(mm, pmd, pmd_pgtable(_pmd));
  2170. spin_unlock(pmd_ptl);
  2171. anon_vma_unlock_write(vma->anon_vma);
  2172. goto out;
  2173. }
  2174. /*
  2175. * All pages are isolated and locked so anon_vma rmap
  2176. * can't run anymore.
  2177. */
  2178. anon_vma_unlock_write(vma->anon_vma);
  2179. __collapse_huge_page_copy(pte, new_page, vma, address, pte_ptl);
  2180. pte_unmap(pte);
  2181. __SetPageUptodate(new_page);
  2182. pgtable = pmd_pgtable(_pmd);
  2183. _pmd = mk_huge_pmd(new_page, vma->vm_page_prot);
  2184. _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma);
  2185. /*
  2186. * spin_lock() below is not the equivalent of smp_wmb(), so
  2187. * this is needed to avoid the copy_huge_page writes to become
  2188. * visible after the set_pmd_at() write.
  2189. */
  2190. smp_wmb();
  2191. spin_lock(pmd_ptl);
  2192. BUG_ON(!pmd_none(*pmd));
  2193. page_add_new_anon_rmap(new_page, vma, address);
  2194. pgtable_trans_huge_deposit(mm, pmd, pgtable);
  2195. set_pmd_at(mm, address, pmd, _pmd);
  2196. update_mmu_cache_pmd(vma, address, pmd);
  2197. spin_unlock(pmd_ptl);
  2198. *hpage = NULL;
  2199. khugepaged_pages_collapsed++;
  2200. out_up_write:
  2201. up_write(&mm->mmap_sem);
  2202. return;
  2203. out:
  2204. mem_cgroup_uncharge_page(new_page);
  2205. goto out_up_write;
  2206. }
  2207. static int khugepaged_scan_pmd(struct mm_struct *mm,
  2208. struct vm_area_struct *vma,
  2209. unsigned long address,
  2210. struct page **hpage)
  2211. {
  2212. pmd_t *pmd;
  2213. pte_t *pte, *_pte;
  2214. int ret = 0, referenced = 0, none = 0;
  2215. struct page *page;
  2216. unsigned long _address;
  2217. spinlock_t *ptl;
  2218. int node = NUMA_NO_NODE;
  2219. VM_BUG_ON(address & ~HPAGE_PMD_MASK);
  2220. pmd = mm_find_pmd(mm, address);
  2221. if (!pmd)
  2222. goto out;
  2223. if (pmd_trans_huge(*pmd))
  2224. goto out;
  2225. memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load));
  2226. pte = pte_offset_map_lock(mm, pmd, address, &ptl);
  2227. for (_address = address, _pte = pte; _pte < pte+HPAGE_PMD_NR;
  2228. _pte++, _address += PAGE_SIZE) {
  2229. pte_t pteval = *_pte;
  2230. if (pte_none(pteval)) {
  2231. if (++none <= khugepaged_max_ptes_none)
  2232. continue;
  2233. else
  2234. goto out_unmap;
  2235. }
  2236. if (!pte_present(pteval) || !pte_write(pteval))
  2237. goto out_unmap;
  2238. page = vm_normal_page(vma, _address, pteval);
  2239. if (unlikely(!page))
  2240. goto out_unmap;
  2241. /*
  2242. * Record which node the original page is from and save this
  2243. * information to khugepaged_node_load[].
  2244. * Khupaged will allocate hugepage from the node has the max
  2245. * hit record.
  2246. */
  2247. node = page_to_nid(page);
  2248. khugepaged_node_load[node]++;
  2249. VM_BUG_ON(PageCompound(page));
  2250. if (!PageLRU(page) || PageLocked(page) || !PageAnon(page))
  2251. goto out_unmap;
  2252. /* cannot use mapcount: can't collapse if there's a gup pin */
  2253. if (page_count(page) != 1)
  2254. goto out_unmap;
  2255. if (pte_young(pteval) || PageReferenced(page) ||
  2256. mmu_notifier_test_young(vma->vm_mm, address))
  2257. referenced = 1;
  2258. }
  2259. if (referenced)
  2260. ret = 1;
  2261. out_unmap:
  2262. pte_unmap_unlock(pte, ptl);
  2263. if (ret) {
  2264. node = khugepaged_find_target_node();
  2265. /* collapse_huge_page will return with the mmap_sem released */
  2266. collapse_huge_page(mm, address, hpage, vma, node);
  2267. }
  2268. out:
  2269. return ret;
  2270. }
  2271. static void collect_mm_slot(struct mm_slot *mm_slot)
  2272. {
  2273. struct mm_struct *mm = mm_slot->mm;
  2274. VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&khugepaged_mm_lock));
  2275. if (khugepaged_test_exit(mm)) {
  2276. /* free mm_slot */
  2277. hash_del(&mm_slot->hash);
  2278. list_del(&mm_slot->mm_node);
  2279. /*
  2280. * Not strictly needed because the mm exited already.
  2281. *
  2282. * clear_bit(MMF_VM_HUGEPAGE, &mm->flags);
  2283. */
  2284. /* khugepaged_mm_lock actually not necessary for the below */
  2285. free_mm_slot(mm_slot);
  2286. mmdrop(mm);
  2287. }
  2288. }
  2289. static unsigned int khugepaged_scan_mm_slot(unsigned int pages,
  2290. struct page **hpage)
  2291. __releases(&khugepaged_mm_lock)
  2292. __acquires(&khugepaged_mm_lock)
  2293. {
  2294. struct mm_slot *mm_slot;
  2295. struct mm_struct *mm;
  2296. struct vm_area_struct *vma;
  2297. int progress = 0;
  2298. VM_BUG_ON(!pages);
  2299. VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&khugepaged_mm_lock));
  2300. if (khugepaged_scan.mm_slot)
  2301. mm_slot = khugepaged_scan.mm_slot;
  2302. else {
  2303. mm_slot = list_entry(khugepaged_scan.mm_head.next,
  2304. struct mm_slot, mm_node);
  2305. khugepaged_scan.address = 0;
  2306. khugepaged_scan.mm_slot = mm_slot;
  2307. }
  2308. spin_unlock(&khugepaged_mm_lock);
  2309. mm = mm_slot->mm;
  2310. down_read(&mm->mmap_sem);
  2311. if (unlikely(khugepaged_test_exit(mm)))
  2312. vma = NULL;
  2313. else
  2314. vma = find_vma(mm, khugepaged_scan.address);
  2315. progress++;
  2316. for (; vma; vma = vma->vm_next) {
  2317. unsigned long hstart, hend;
  2318. cond_resched();
  2319. if (unlikely(khugepaged_test_exit(mm))) {
  2320. progress++;
  2321. break;
  2322. }
  2323. if (!hugepage_vma_check(vma)) {
  2324. skip:
  2325. progress++;
  2326. continue;
  2327. }
  2328. hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
  2329. hend = vma->vm_end & HPAGE_PMD_MASK;
  2330. if (hstart >= hend)
  2331. goto skip;
  2332. if (khugepaged_scan.address > hend)
  2333. goto skip;
  2334. if (khugepaged_scan.address < hstart)
  2335. khugepaged_scan.address = hstart;
  2336. VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK);
  2337. while (khugepaged_scan.address < hend) {
  2338. int ret;
  2339. cond_resched();
  2340. if (unlikely(khugepaged_test_exit(mm)))
  2341. goto breakouterloop;
  2342. VM_BUG_ON(khugepaged_scan.address < hstart ||
  2343. khugepaged_scan.address + HPAGE_PMD_SIZE >
  2344. hend);
  2345. ret = khugepaged_scan_pmd(mm, vma,
  2346. khugepaged_scan.address,
  2347. hpage);
  2348. /* move to next address */
  2349. khugepaged_scan.address += HPAGE_PMD_SIZE;
  2350. progress += HPAGE_PMD_NR;
  2351. if (ret)
  2352. /* we released mmap_sem so break loop */
  2353. goto breakouterloop_mmap_sem;
  2354. if (progress >= pages)
  2355. goto breakouterloop;
  2356. }
  2357. }
  2358. breakouterloop:
  2359. up_read(&mm->mmap_sem); /* exit_mmap will destroy ptes after this */
  2360. breakouterloop_mmap_sem:
  2361. spin_lock(&khugepaged_mm_lock);
  2362. VM_BUG_ON(khugepaged_scan.mm_slot != mm_slot);
  2363. /*
  2364. * Release the current mm_slot if this mm is about to die, or
  2365. * if we scanned all vmas of this mm.
  2366. */
  2367. if (khugepaged_test_exit(mm) || !vma) {
  2368. /*
  2369. * Make sure that if mm_users is reaching zero while
  2370. * khugepaged runs here, khugepaged_exit will find
  2371. * mm_slot not pointing to the exiting mm.
  2372. */
  2373. if (mm_slot->mm_node.next != &khugepaged_scan.mm_head) {
  2374. khugepaged_scan.mm_slot = list_entry(
  2375. mm_slot->mm_node.next,
  2376. struct mm_slot, mm_node);
  2377. khugepaged_scan.address = 0;
  2378. } else {
  2379. khugepaged_scan.mm_slot = NULL;
  2380. khugepaged_full_scans++;
  2381. }
  2382. collect_mm_slot(mm_slot);
  2383. }
  2384. return progress;
  2385. }
  2386. static int khugepaged_has_work(void)
  2387. {
  2388. return !list_empty(&khugepaged_scan.mm_head) &&
  2389. khugepaged_enabled();
  2390. }
  2391. static int khugepaged_wait_event(void)
  2392. {
  2393. return !list_empty(&khugepaged_scan.mm_head) ||
  2394. kthread_should_stop();
  2395. }
  2396. static void khugepaged_do_scan(void)
  2397. {
  2398. struct page *hpage = NULL;
  2399. unsigned int progress = 0, pass_through_head = 0;
  2400. unsigned int pages = khugepaged_pages_to_scan;
  2401. bool wait = true;
  2402. barrier(); /* write khugepaged_pages_to_scan to local stack */
  2403. while (progress < pages) {
  2404. if (!khugepaged_prealloc_page(&hpage, &wait))
  2405. break;
  2406. cond_resched();
  2407. if (unlikely(kthread_should_stop() || freezing(current)))
  2408. break;
  2409. spin_lock(&khugepaged_mm_lock);
  2410. if (!khugepaged_scan.mm_slot)
  2411. pass_through_head++;
  2412. if (khugepaged_has_work() &&
  2413. pass_through_head < 2)
  2414. progress += khugepaged_scan_mm_slot(pages - progress,
  2415. &hpage);
  2416. else
  2417. progress = pages;
  2418. spin_unlock(&khugepaged_mm_lock);
  2419. }
  2420. if (!IS_ERR_OR_NULL(hpage))
  2421. put_page(hpage);
  2422. }
  2423. static void khugepaged_wait_work(void)
  2424. {
  2425. try_to_freeze();
  2426. if (khugepaged_has_work()) {
  2427. if (!khugepaged_scan_sleep_millisecs)
  2428. return;
  2429. wait_event_freezable_timeout(khugepaged_wait,
  2430. kthread_should_stop(),
  2431. msecs_to_jiffies(khugepaged_scan_sleep_millisecs));
  2432. return;
  2433. }
  2434. if (khugepaged_enabled())
  2435. wait_event_freezable(khugepaged_wait, khugepaged_wait_event());
  2436. }
  2437. static int khugepaged(void *none)
  2438. {
  2439. struct mm_slot *mm_slot;
  2440. set_freezable();
  2441. set_user_nice(current, 19);
  2442. while (!kthread_should_stop()) {
  2443. khugepaged_do_scan();
  2444. khugepaged_wait_work();
  2445. }
  2446. spin_lock(&khugepaged_mm_lock);
  2447. mm_slot = khugepaged_scan.mm_slot;
  2448. khugepaged_scan.mm_slot = NULL;
  2449. if (mm_slot)
  2450. collect_mm_slot(mm_slot);
  2451. spin_unlock(&khugepaged_mm_lock);
  2452. return 0;
  2453. }
  2454. static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
  2455. unsigned long haddr, pmd_t *pmd)
  2456. {
  2457. struct mm_struct *mm = vma->vm_mm;
  2458. pgtable_t pgtable;
  2459. pmd_t _pmd;
  2460. int i;
  2461. pmdp_clear_flush(vma, haddr, pmd);
  2462. /* leave pmd empty until pte is filled */
  2463. pgtable = pgtable_trans_huge_withdraw(mm, pmd);
  2464. pmd_populate(mm, &_pmd, pgtable);
  2465. for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
  2466. pte_t *pte, entry;
  2467. entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot);
  2468. entry = pte_mkspecial(entry);
  2469. pte = pte_offset_map(&_pmd, haddr);
  2470. VM_BUG_ON(!pte_none(*pte));
  2471. set_pte_at(mm, haddr, pte, entry);
  2472. pte_unmap(pte);
  2473. }
  2474. smp_wmb(); /* make pte visible before pmd */
  2475. pmd_populate(mm, pmd, pgtable);
  2476. put_huge_zero_page();
  2477. }
  2478. void __split_huge_page_pmd(struct vm_area_struct *vma, unsigned long address,
  2479. pmd_t *pmd)
  2480. {
  2481. spinlock_t *ptl;
  2482. struct page *page;
  2483. struct mm_struct *mm = vma->vm_mm;
  2484. unsigned long haddr = address & HPAGE_PMD_MASK;
  2485. unsigned long mmun_start; /* For mmu_notifiers */
  2486. unsigned long mmun_end; /* For mmu_notifiers */
  2487. BUG_ON(vma->vm_start > haddr || vma->vm_end < haddr + HPAGE_PMD_SIZE);
  2488. mmun_start = haddr;
  2489. mmun_end = haddr + HPAGE_PMD_SIZE;
  2490. again:
  2491. mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
  2492. ptl = pmd_lock(mm, pmd);
  2493. if (unlikely(!pmd_trans_huge(*pmd))) {
  2494. spin_unlock(ptl);
  2495. mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
  2496. return;
  2497. }
  2498. if (is_huge_zero_pmd(*pmd)) {
  2499. __split_huge_zero_page_pmd(vma, haddr, pmd);
  2500. spin_unlock(ptl);
  2501. mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
  2502. return;
  2503. }
  2504. page = pmd_page(*pmd);
  2505. VM_BUG_ON(!page_count(page));
  2506. get_page(page);
  2507. spin_unlock(ptl);
  2508. mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
  2509. split_huge_page(page);
  2510. put_page(page);
  2511. /*
  2512. * We don't always have down_write of mmap_sem here: a racing
  2513. * do_huge_pmd_wp_page() might have copied-on-write to another
  2514. * huge page before our split_huge_page() got the anon_vma lock.
  2515. */
  2516. if (unlikely(pmd_trans_huge(*pmd)))
  2517. goto again;
  2518. }
  2519. void split_huge_page_pmd_mm(struct mm_struct *mm, unsigned long address,
  2520. pmd_t *pmd)
  2521. {
  2522. struct vm_area_struct *vma;
  2523. vma = find_vma(mm, address);
  2524. BUG_ON(vma == NULL);
  2525. split_huge_page_pmd(vma, address, pmd);
  2526. }
  2527. static void split_huge_page_address(struct mm_struct *mm,
  2528. unsigned long address)
  2529. {
  2530. pmd_t *pmd;
  2531. VM_BUG_ON(!(address & ~HPAGE_PMD_MASK));
  2532. pmd = mm_find_pmd(mm, address);
  2533. if (!pmd)
  2534. return;
  2535. /*
  2536. * Caller holds the mmap_sem write mode, so a huge pmd cannot
  2537. * materialize from under us.
  2538. */
  2539. split_huge_page_pmd_mm(mm, address, pmd);
  2540. }
  2541. void __vma_adjust_trans_huge(struct vm_area_struct *vma,
  2542. unsigned long start,
  2543. unsigned long end,
  2544. long adjust_next)
  2545. {
  2546. /*
  2547. * If the new start address isn't hpage aligned and it could
  2548. * previously contain an hugepage: check if we need to split
  2549. * an huge pmd.
  2550. */
  2551. if (start & ~HPAGE_PMD_MASK &&
  2552. (start & HPAGE_PMD_MASK) >= vma->vm_start &&
  2553. (start & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)
  2554. split_huge_page_address(vma->vm_mm, start);
  2555. /*
  2556. * If the new end address isn't hpage aligned and it could
  2557. * previously contain an hugepage: check if we need to split
  2558. * an huge pmd.
  2559. */
  2560. if (end & ~HPAGE_PMD_MASK &&
  2561. (end & HPAGE_PMD_MASK) >= vma->vm_start &&
  2562. (end & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)
  2563. split_huge_page_address(vma->vm_mm, end);
  2564. /*
  2565. * If we're also updating the vma->vm_next->vm_start, if the new
  2566. * vm_next->vm_start isn't page aligned and it could previously
  2567. * contain an hugepage: check if we need to split an huge pmd.
  2568. */
  2569. if (adjust_next > 0) {
  2570. struct vm_area_struct *next = vma->vm_next;
  2571. unsigned long nstart = next->vm_start;
  2572. nstart += adjust_next << PAGE_SHIFT;
  2573. if (nstart & ~HPAGE_PMD_MASK &&
  2574. (nstart & HPAGE_PMD_MASK) >= next->vm_start &&
  2575. (nstart & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= next->vm_end)
  2576. split_huge_page_address(next->vm_mm, nstart);
  2577. }
  2578. }