hugetlb.c 92 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542
  1. /*
  2. * Generic hugetlb support.
  3. * (C) Nadia Yvette Chambers, April 2004
  4. */
  5. #include <linux/list.h>
  6. #include <linux/init.h>
  7. #include <linux/module.h>
  8. #include <linux/mm.h>
  9. #include <linux/seq_file.h>
  10. #include <linux/sysctl.h>
  11. #include <linux/highmem.h>
  12. #include <linux/mmu_notifier.h>
  13. #include <linux/nodemask.h>
  14. #include <linux/pagemap.h>
  15. #include <linux/mempolicy.h>
  16. #include <linux/cpuset.h>
  17. #include <linux/mutex.h>
  18. #include <linux/bootmem.h>
  19. #include <linux/sysfs.h>
  20. #include <linux/slab.h>
  21. #include <linux/rmap.h>
  22. #include <linux/swap.h>
  23. #include <linux/swapops.h>
  24. #include <linux/page-isolation.h>
  25. #include <asm/page.h>
  26. #include <asm/pgtable.h>
  27. #include <asm/tlb.h>
  28. #include <linux/io.h>
  29. #include <linux/hugetlb.h>
  30. #include <linux/hugetlb_cgroup.h>
  31. #include <linux/node.h>
  32. #include "internal.h"
  33. const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
  34. unsigned long hugepages_treat_as_movable;
  35. int hugetlb_max_hstate __read_mostly;
  36. unsigned int default_hstate_idx;
  37. struct hstate hstates[HUGE_MAX_HSTATE];
  38. __initdata LIST_HEAD(huge_boot_pages);
  39. /* for command line parsing */
  40. static struct hstate * __initdata parsed_hstate;
  41. static unsigned long __initdata default_hstate_max_huge_pages;
  42. static unsigned long __initdata default_hstate_size;
  43. /*
  44. * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
  45. * free_huge_pages, and surplus_huge_pages.
  46. */
  47. DEFINE_SPINLOCK(hugetlb_lock);
  48. static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
  49. {
  50. bool free = (spool->count == 0) && (spool->used_hpages == 0);
  51. spin_unlock(&spool->lock);
  52. /* If no pages are used, and no other handles to the subpool
  53. * remain, free the subpool the subpool remain */
  54. if (free)
  55. kfree(spool);
  56. }
  57. struct hugepage_subpool *hugepage_new_subpool(long nr_blocks)
  58. {
  59. struct hugepage_subpool *spool;
  60. spool = kmalloc(sizeof(*spool), GFP_KERNEL);
  61. if (!spool)
  62. return NULL;
  63. spin_lock_init(&spool->lock);
  64. spool->count = 1;
  65. spool->max_hpages = nr_blocks;
  66. spool->used_hpages = 0;
  67. return spool;
  68. }
  69. void hugepage_put_subpool(struct hugepage_subpool *spool)
  70. {
  71. spin_lock(&spool->lock);
  72. BUG_ON(!spool->count);
  73. spool->count--;
  74. unlock_or_release_subpool(spool);
  75. }
  76. static int hugepage_subpool_get_pages(struct hugepage_subpool *spool,
  77. long delta)
  78. {
  79. int ret = 0;
  80. if (!spool)
  81. return 0;
  82. spin_lock(&spool->lock);
  83. if ((spool->used_hpages + delta) <= spool->max_hpages) {
  84. spool->used_hpages += delta;
  85. } else {
  86. ret = -ENOMEM;
  87. }
  88. spin_unlock(&spool->lock);
  89. return ret;
  90. }
  91. static void hugepage_subpool_put_pages(struct hugepage_subpool *spool,
  92. long delta)
  93. {
  94. if (!spool)
  95. return;
  96. spin_lock(&spool->lock);
  97. spool->used_hpages -= delta;
  98. /* If hugetlbfs_put_super couldn't free spool due to
  99. * an outstanding quota reference, free it now. */
  100. unlock_or_release_subpool(spool);
  101. }
  102. static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
  103. {
  104. return HUGETLBFS_SB(inode->i_sb)->spool;
  105. }
  106. static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
  107. {
  108. return subpool_inode(file_inode(vma->vm_file));
  109. }
  110. /*
  111. * Region tracking -- allows tracking of reservations and instantiated pages
  112. * across the pages in a mapping.
  113. *
  114. * The region data structures are protected by a combination of the mmap_sem
  115. * and the hugetlb_instantiation_mutex. To access or modify a region the caller
  116. * must either hold the mmap_sem for write, or the mmap_sem for read and
  117. * the hugetlb_instantiation_mutex:
  118. *
  119. * down_write(&mm->mmap_sem);
  120. * or
  121. * down_read(&mm->mmap_sem);
  122. * mutex_lock(&hugetlb_instantiation_mutex);
  123. */
  124. struct file_region {
  125. struct list_head link;
  126. long from;
  127. long to;
  128. };
  129. static long region_add(struct list_head *head, long f, long t)
  130. {
  131. struct file_region *rg, *nrg, *trg;
  132. /* Locate the region we are either in or before. */
  133. list_for_each_entry(rg, head, link)
  134. if (f <= rg->to)
  135. break;
  136. /* Round our left edge to the current segment if it encloses us. */
  137. if (f > rg->from)
  138. f = rg->from;
  139. /* Check for and consume any regions we now overlap with. */
  140. nrg = rg;
  141. list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
  142. if (&rg->link == head)
  143. break;
  144. if (rg->from > t)
  145. break;
  146. /* If this area reaches higher then extend our area to
  147. * include it completely. If this is not the first area
  148. * which we intend to reuse, free it. */
  149. if (rg->to > t)
  150. t = rg->to;
  151. if (rg != nrg) {
  152. list_del(&rg->link);
  153. kfree(rg);
  154. }
  155. }
  156. nrg->from = f;
  157. nrg->to = t;
  158. return 0;
  159. }
  160. static long region_chg(struct list_head *head, long f, long t)
  161. {
  162. struct file_region *rg, *nrg;
  163. long chg = 0;
  164. /* Locate the region we are before or in. */
  165. list_for_each_entry(rg, head, link)
  166. if (f <= rg->to)
  167. break;
  168. /* If we are below the current region then a new region is required.
  169. * Subtle, allocate a new region at the position but make it zero
  170. * size such that we can guarantee to record the reservation. */
  171. if (&rg->link == head || t < rg->from) {
  172. nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
  173. if (!nrg)
  174. return -ENOMEM;
  175. nrg->from = f;
  176. nrg->to = f;
  177. INIT_LIST_HEAD(&nrg->link);
  178. list_add(&nrg->link, rg->link.prev);
  179. return t - f;
  180. }
  181. /* Round our left edge to the current segment if it encloses us. */
  182. if (f > rg->from)
  183. f = rg->from;
  184. chg = t - f;
  185. /* Check for and consume any regions we now overlap with. */
  186. list_for_each_entry(rg, rg->link.prev, link) {
  187. if (&rg->link == head)
  188. break;
  189. if (rg->from > t)
  190. return chg;
  191. /* We overlap with this area, if it extends further than
  192. * us then we must extend ourselves. Account for its
  193. * existing reservation. */
  194. if (rg->to > t) {
  195. chg += rg->to - t;
  196. t = rg->to;
  197. }
  198. chg -= rg->to - rg->from;
  199. }
  200. return chg;
  201. }
  202. static long region_truncate(struct list_head *head, long end)
  203. {
  204. struct file_region *rg, *trg;
  205. long chg = 0;
  206. /* Locate the region we are either in or before. */
  207. list_for_each_entry(rg, head, link)
  208. if (end <= rg->to)
  209. break;
  210. if (&rg->link == head)
  211. return 0;
  212. /* If we are in the middle of a region then adjust it. */
  213. if (end > rg->from) {
  214. chg = rg->to - end;
  215. rg->to = end;
  216. rg = list_entry(rg->link.next, typeof(*rg), link);
  217. }
  218. /* Drop any remaining regions. */
  219. list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
  220. if (&rg->link == head)
  221. break;
  222. chg += rg->to - rg->from;
  223. list_del(&rg->link);
  224. kfree(rg);
  225. }
  226. return chg;
  227. }
  228. static long region_count(struct list_head *head, long f, long t)
  229. {
  230. struct file_region *rg;
  231. long chg = 0;
  232. /* Locate each segment we overlap with, and count that overlap. */
  233. list_for_each_entry(rg, head, link) {
  234. long seg_from;
  235. long seg_to;
  236. if (rg->to <= f)
  237. continue;
  238. if (rg->from >= t)
  239. break;
  240. seg_from = max(rg->from, f);
  241. seg_to = min(rg->to, t);
  242. chg += seg_to - seg_from;
  243. }
  244. return chg;
  245. }
  246. /*
  247. * Convert the address within this vma to the page offset within
  248. * the mapping, in pagecache page units; huge pages here.
  249. */
  250. static pgoff_t vma_hugecache_offset(struct hstate *h,
  251. struct vm_area_struct *vma, unsigned long address)
  252. {
  253. return ((address - vma->vm_start) >> huge_page_shift(h)) +
  254. (vma->vm_pgoff >> huge_page_order(h));
  255. }
  256. pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
  257. unsigned long address)
  258. {
  259. return vma_hugecache_offset(hstate_vma(vma), vma, address);
  260. }
  261. /*
  262. * Return the size of the pages allocated when backing a VMA. In the majority
  263. * cases this will be same size as used by the page table entries.
  264. */
  265. unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
  266. {
  267. struct hstate *hstate;
  268. if (!is_vm_hugetlb_page(vma))
  269. return PAGE_SIZE;
  270. hstate = hstate_vma(vma);
  271. return 1UL << huge_page_shift(hstate);
  272. }
  273. EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
  274. /*
  275. * Return the page size being used by the MMU to back a VMA. In the majority
  276. * of cases, the page size used by the kernel matches the MMU size. On
  277. * architectures where it differs, an architecture-specific version of this
  278. * function is required.
  279. */
  280. #ifndef vma_mmu_pagesize
  281. unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
  282. {
  283. return vma_kernel_pagesize(vma);
  284. }
  285. #endif
  286. /*
  287. * Flags for MAP_PRIVATE reservations. These are stored in the bottom
  288. * bits of the reservation map pointer, which are always clear due to
  289. * alignment.
  290. */
  291. #define HPAGE_RESV_OWNER (1UL << 0)
  292. #define HPAGE_RESV_UNMAPPED (1UL << 1)
  293. #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
  294. /*
  295. * These helpers are used to track how many pages are reserved for
  296. * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
  297. * is guaranteed to have their future faults succeed.
  298. *
  299. * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
  300. * the reserve counters are updated with the hugetlb_lock held. It is safe
  301. * to reset the VMA at fork() time as it is not in use yet and there is no
  302. * chance of the global counters getting corrupted as a result of the values.
  303. *
  304. * The private mapping reservation is represented in a subtly different
  305. * manner to a shared mapping. A shared mapping has a region map associated
  306. * with the underlying file, this region map represents the backing file
  307. * pages which have ever had a reservation assigned which this persists even
  308. * after the page is instantiated. A private mapping has a region map
  309. * associated with the original mmap which is attached to all VMAs which
  310. * reference it, this region map represents those offsets which have consumed
  311. * reservation ie. where pages have been instantiated.
  312. */
  313. static unsigned long get_vma_private_data(struct vm_area_struct *vma)
  314. {
  315. return (unsigned long)vma->vm_private_data;
  316. }
  317. static void set_vma_private_data(struct vm_area_struct *vma,
  318. unsigned long value)
  319. {
  320. vma->vm_private_data = (void *)value;
  321. }
  322. struct resv_map {
  323. struct kref refs;
  324. struct list_head regions;
  325. };
  326. static struct resv_map *resv_map_alloc(void)
  327. {
  328. struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
  329. if (!resv_map)
  330. return NULL;
  331. kref_init(&resv_map->refs);
  332. INIT_LIST_HEAD(&resv_map->regions);
  333. return resv_map;
  334. }
  335. static void resv_map_release(struct kref *ref)
  336. {
  337. struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
  338. /* Clear out any active regions before we release the map. */
  339. region_truncate(&resv_map->regions, 0);
  340. kfree(resv_map);
  341. }
  342. static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
  343. {
  344. VM_BUG_ON(!is_vm_hugetlb_page(vma));
  345. if (!(vma->vm_flags & VM_MAYSHARE))
  346. return (struct resv_map *)(get_vma_private_data(vma) &
  347. ~HPAGE_RESV_MASK);
  348. return NULL;
  349. }
  350. static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
  351. {
  352. VM_BUG_ON(!is_vm_hugetlb_page(vma));
  353. VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
  354. set_vma_private_data(vma, (get_vma_private_data(vma) &
  355. HPAGE_RESV_MASK) | (unsigned long)map);
  356. }
  357. static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
  358. {
  359. VM_BUG_ON(!is_vm_hugetlb_page(vma));
  360. VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
  361. set_vma_private_data(vma, get_vma_private_data(vma) | flags);
  362. }
  363. static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
  364. {
  365. VM_BUG_ON(!is_vm_hugetlb_page(vma));
  366. return (get_vma_private_data(vma) & flag) != 0;
  367. }
  368. /* Reset counters to 0 and clear all HPAGE_RESV_* flags */
  369. void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
  370. {
  371. VM_BUG_ON(!is_vm_hugetlb_page(vma));
  372. if (!(vma->vm_flags & VM_MAYSHARE))
  373. vma->vm_private_data = (void *)0;
  374. }
  375. /* Returns true if the VMA has associated reserve pages */
  376. static int vma_has_reserves(struct vm_area_struct *vma, long chg)
  377. {
  378. if (vma->vm_flags & VM_NORESERVE) {
  379. /*
  380. * This address is already reserved by other process(chg == 0),
  381. * so, we should decrement reserved count. Without decrementing,
  382. * reserve count remains after releasing inode, because this
  383. * allocated page will go into page cache and is regarded as
  384. * coming from reserved pool in releasing step. Currently, we
  385. * don't have any other solution to deal with this situation
  386. * properly, so add work-around here.
  387. */
  388. if (vma->vm_flags & VM_MAYSHARE && chg == 0)
  389. return 1;
  390. else
  391. return 0;
  392. }
  393. /* Shared mappings always use reserves */
  394. if (vma->vm_flags & VM_MAYSHARE)
  395. return 1;
  396. /*
  397. * Only the process that called mmap() has reserves for
  398. * private mappings.
  399. */
  400. if (is_vma_resv_set(vma, HPAGE_RESV_OWNER))
  401. return 1;
  402. return 0;
  403. }
  404. static void enqueue_huge_page(struct hstate *h, struct page *page)
  405. {
  406. int nid = page_to_nid(page);
  407. list_move(&page->lru, &h->hugepage_freelists[nid]);
  408. h->free_huge_pages++;
  409. h->free_huge_pages_node[nid]++;
  410. }
  411. static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
  412. {
  413. struct page *page;
  414. list_for_each_entry(page, &h->hugepage_freelists[nid], lru)
  415. if (!is_migrate_isolate_page(page))
  416. break;
  417. /*
  418. * if 'non-isolated free hugepage' not found on the list,
  419. * the allocation fails.
  420. */
  421. if (&h->hugepage_freelists[nid] == &page->lru)
  422. return NULL;
  423. list_move(&page->lru, &h->hugepage_activelist);
  424. set_page_refcounted(page);
  425. h->free_huge_pages--;
  426. h->free_huge_pages_node[nid]--;
  427. return page;
  428. }
  429. /* Movability of hugepages depends on migration support. */
  430. static inline gfp_t htlb_alloc_mask(struct hstate *h)
  431. {
  432. if (hugepages_treat_as_movable || hugepage_migration_support(h))
  433. return GFP_HIGHUSER_MOVABLE;
  434. else
  435. return GFP_HIGHUSER;
  436. }
  437. static struct page *dequeue_huge_page_vma(struct hstate *h,
  438. struct vm_area_struct *vma,
  439. unsigned long address, int avoid_reserve,
  440. long chg)
  441. {
  442. struct page *page = NULL;
  443. struct mempolicy *mpol;
  444. nodemask_t *nodemask;
  445. struct zonelist *zonelist;
  446. struct zone *zone;
  447. struct zoneref *z;
  448. unsigned int cpuset_mems_cookie;
  449. /*
  450. * A child process with MAP_PRIVATE mappings created by their parent
  451. * have no page reserves. This check ensures that reservations are
  452. * not "stolen". The child may still get SIGKILLed
  453. */
  454. if (!vma_has_reserves(vma, chg) &&
  455. h->free_huge_pages - h->resv_huge_pages == 0)
  456. goto err;
  457. /* If reserves cannot be used, ensure enough pages are in the pool */
  458. if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
  459. goto err;
  460. retry_cpuset:
  461. cpuset_mems_cookie = get_mems_allowed();
  462. zonelist = huge_zonelist(vma, address,
  463. htlb_alloc_mask(h), &mpol, &nodemask);
  464. for_each_zone_zonelist_nodemask(zone, z, zonelist,
  465. MAX_NR_ZONES - 1, nodemask) {
  466. if (cpuset_zone_allowed_softwall(zone, htlb_alloc_mask(h))) {
  467. page = dequeue_huge_page_node(h, zone_to_nid(zone));
  468. if (page) {
  469. if (avoid_reserve)
  470. break;
  471. if (!vma_has_reserves(vma, chg))
  472. break;
  473. SetPagePrivate(page);
  474. h->resv_huge_pages--;
  475. break;
  476. }
  477. }
  478. }
  479. mpol_cond_put(mpol);
  480. if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
  481. goto retry_cpuset;
  482. return page;
  483. err:
  484. return NULL;
  485. }
  486. static void update_and_free_page(struct hstate *h, struct page *page)
  487. {
  488. int i;
  489. VM_BUG_ON(h->order >= MAX_ORDER);
  490. h->nr_huge_pages--;
  491. h->nr_huge_pages_node[page_to_nid(page)]--;
  492. for (i = 0; i < pages_per_huge_page(h); i++) {
  493. page[i].flags &= ~(1 << PG_locked | 1 << PG_error |
  494. 1 << PG_referenced | 1 << PG_dirty |
  495. 1 << PG_active | 1 << PG_reserved |
  496. 1 << PG_private | 1 << PG_writeback);
  497. }
  498. VM_BUG_ON(hugetlb_cgroup_from_page(page));
  499. set_compound_page_dtor(page, NULL);
  500. set_page_refcounted(page);
  501. arch_release_hugepage(page);
  502. __free_pages(page, huge_page_order(h));
  503. }
  504. struct hstate *size_to_hstate(unsigned long size)
  505. {
  506. struct hstate *h;
  507. for_each_hstate(h) {
  508. if (huge_page_size(h) == size)
  509. return h;
  510. }
  511. return NULL;
  512. }
  513. static void free_huge_page(struct page *page)
  514. {
  515. /*
  516. * Can't pass hstate in here because it is called from the
  517. * compound page destructor.
  518. */
  519. struct hstate *h = page_hstate(page);
  520. int nid = page_to_nid(page);
  521. struct hugepage_subpool *spool =
  522. (struct hugepage_subpool *)page_private(page);
  523. bool restore_reserve;
  524. set_page_private(page, 0);
  525. page->mapping = NULL;
  526. BUG_ON(page_count(page));
  527. BUG_ON(page_mapcount(page));
  528. restore_reserve = PagePrivate(page);
  529. ClearPagePrivate(page);
  530. spin_lock(&hugetlb_lock);
  531. hugetlb_cgroup_uncharge_page(hstate_index(h),
  532. pages_per_huge_page(h), page);
  533. if (restore_reserve)
  534. h->resv_huge_pages++;
  535. if (h->surplus_huge_pages_node[nid] && huge_page_order(h) < MAX_ORDER) {
  536. /* remove the page from active list */
  537. list_del(&page->lru);
  538. update_and_free_page(h, page);
  539. h->surplus_huge_pages--;
  540. h->surplus_huge_pages_node[nid]--;
  541. } else {
  542. arch_clear_hugepage_flags(page);
  543. enqueue_huge_page(h, page);
  544. }
  545. spin_unlock(&hugetlb_lock);
  546. hugepage_subpool_put_pages(spool, 1);
  547. }
  548. static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
  549. {
  550. INIT_LIST_HEAD(&page->lru);
  551. set_compound_page_dtor(page, free_huge_page);
  552. spin_lock(&hugetlb_lock);
  553. set_hugetlb_cgroup(page, NULL);
  554. h->nr_huge_pages++;
  555. h->nr_huge_pages_node[nid]++;
  556. spin_unlock(&hugetlb_lock);
  557. put_page(page); /* free it into the hugepage allocator */
  558. }
  559. static void prep_compound_gigantic_page(struct page *page, unsigned long order)
  560. {
  561. int i;
  562. int nr_pages = 1 << order;
  563. struct page *p = page + 1;
  564. /* we rely on prep_new_huge_page to set the destructor */
  565. set_compound_order(page, order);
  566. __SetPageHead(page);
  567. __ClearPageReserved(page);
  568. for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
  569. __SetPageTail(p);
  570. /*
  571. * For gigantic hugepages allocated through bootmem at
  572. * boot, it's safer to be consistent with the not-gigantic
  573. * hugepages and clear the PG_reserved bit from all tail pages
  574. * too. Otherwse drivers using get_user_pages() to access tail
  575. * pages may get the reference counting wrong if they see
  576. * PG_reserved set on a tail page (despite the head page not
  577. * having PG_reserved set). Enforcing this consistency between
  578. * head and tail pages allows drivers to optimize away a check
  579. * on the head page when they need know if put_page() is needed
  580. * after get_user_pages().
  581. */
  582. __ClearPageReserved(p);
  583. set_page_count(p, 0);
  584. p->first_page = page;
  585. }
  586. }
  587. /*
  588. * PageHuge() only returns true for hugetlbfs pages, but not for normal or
  589. * transparent huge pages. See the PageTransHuge() documentation for more
  590. * details.
  591. */
  592. int PageHuge(struct page *page)
  593. {
  594. compound_page_dtor *dtor;
  595. if (!PageCompound(page))
  596. return 0;
  597. page = compound_head(page);
  598. dtor = get_compound_page_dtor(page);
  599. return dtor == free_huge_page;
  600. }
  601. EXPORT_SYMBOL_GPL(PageHuge);
  602. /*
  603. * PageHeadHuge() only returns true for hugetlbfs head page, but not for
  604. * normal or transparent huge pages.
  605. */
  606. int PageHeadHuge(struct page *page_head)
  607. {
  608. compound_page_dtor *dtor;
  609. if (!PageHead(page_head))
  610. return 0;
  611. dtor = get_compound_page_dtor(page_head);
  612. return dtor == free_huge_page;
  613. }
  614. EXPORT_SYMBOL_GPL(PageHeadHuge);
  615. pgoff_t __basepage_index(struct page *page)
  616. {
  617. struct page *page_head = compound_head(page);
  618. pgoff_t index = page_index(page_head);
  619. unsigned long compound_idx;
  620. if (!PageHuge(page_head))
  621. return page_index(page);
  622. if (compound_order(page_head) >= MAX_ORDER)
  623. compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
  624. else
  625. compound_idx = page - page_head;
  626. return (index << compound_order(page_head)) + compound_idx;
  627. }
  628. static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
  629. {
  630. struct page *page;
  631. if (h->order >= MAX_ORDER)
  632. return NULL;
  633. page = alloc_pages_exact_node(nid,
  634. htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
  635. __GFP_REPEAT|__GFP_NOWARN,
  636. huge_page_order(h));
  637. if (page) {
  638. if (arch_prepare_hugepage(page)) {
  639. __free_pages(page, huge_page_order(h));
  640. return NULL;
  641. }
  642. prep_new_huge_page(h, page, nid);
  643. }
  644. return page;
  645. }
  646. /*
  647. * common helper functions for hstate_next_node_to_{alloc|free}.
  648. * We may have allocated or freed a huge page based on a different
  649. * nodes_allowed previously, so h->next_node_to_{alloc|free} might
  650. * be outside of *nodes_allowed. Ensure that we use an allowed
  651. * node for alloc or free.
  652. */
  653. static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
  654. {
  655. nid = next_node(nid, *nodes_allowed);
  656. if (nid == MAX_NUMNODES)
  657. nid = first_node(*nodes_allowed);
  658. VM_BUG_ON(nid >= MAX_NUMNODES);
  659. return nid;
  660. }
  661. static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
  662. {
  663. if (!node_isset(nid, *nodes_allowed))
  664. nid = next_node_allowed(nid, nodes_allowed);
  665. return nid;
  666. }
  667. /*
  668. * returns the previously saved node ["this node"] from which to
  669. * allocate a persistent huge page for the pool and advance the
  670. * next node from which to allocate, handling wrap at end of node
  671. * mask.
  672. */
  673. static int hstate_next_node_to_alloc(struct hstate *h,
  674. nodemask_t *nodes_allowed)
  675. {
  676. int nid;
  677. VM_BUG_ON(!nodes_allowed);
  678. nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
  679. h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
  680. return nid;
  681. }
  682. /*
  683. * helper for free_pool_huge_page() - return the previously saved
  684. * node ["this node"] from which to free a huge page. Advance the
  685. * next node id whether or not we find a free huge page to free so
  686. * that the next attempt to free addresses the next node.
  687. */
  688. static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
  689. {
  690. int nid;
  691. VM_BUG_ON(!nodes_allowed);
  692. nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
  693. h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
  694. return nid;
  695. }
  696. #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask) \
  697. for (nr_nodes = nodes_weight(*mask); \
  698. nr_nodes > 0 && \
  699. ((node = hstate_next_node_to_alloc(hs, mask)) || 1); \
  700. nr_nodes--)
  701. #define for_each_node_mask_to_free(hs, nr_nodes, node, mask) \
  702. for (nr_nodes = nodes_weight(*mask); \
  703. nr_nodes > 0 && \
  704. ((node = hstate_next_node_to_free(hs, mask)) || 1); \
  705. nr_nodes--)
  706. static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
  707. {
  708. struct page *page;
  709. int nr_nodes, node;
  710. int ret = 0;
  711. for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
  712. page = alloc_fresh_huge_page_node(h, node);
  713. if (page) {
  714. ret = 1;
  715. break;
  716. }
  717. }
  718. if (ret)
  719. count_vm_event(HTLB_BUDDY_PGALLOC);
  720. else
  721. count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
  722. return ret;
  723. }
  724. /*
  725. * Free huge page from pool from next node to free.
  726. * Attempt to keep persistent huge pages more or less
  727. * balanced over allowed nodes.
  728. * Called with hugetlb_lock locked.
  729. */
  730. static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
  731. bool acct_surplus)
  732. {
  733. int nr_nodes, node;
  734. int ret = 0;
  735. for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
  736. /*
  737. * If we're returning unused surplus pages, only examine
  738. * nodes with surplus pages.
  739. */
  740. if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
  741. !list_empty(&h->hugepage_freelists[node])) {
  742. struct page *page =
  743. list_entry(h->hugepage_freelists[node].next,
  744. struct page, lru);
  745. list_del(&page->lru);
  746. h->free_huge_pages--;
  747. h->free_huge_pages_node[node]--;
  748. if (acct_surplus) {
  749. h->surplus_huge_pages--;
  750. h->surplus_huge_pages_node[node]--;
  751. }
  752. update_and_free_page(h, page);
  753. ret = 1;
  754. break;
  755. }
  756. }
  757. return ret;
  758. }
  759. /*
  760. * Dissolve a given free hugepage into free buddy pages. This function does
  761. * nothing for in-use (including surplus) hugepages.
  762. */
  763. static void dissolve_free_huge_page(struct page *page)
  764. {
  765. spin_lock(&hugetlb_lock);
  766. if (PageHuge(page) && !page_count(page)) {
  767. struct hstate *h = page_hstate(page);
  768. int nid = page_to_nid(page);
  769. list_del(&page->lru);
  770. h->free_huge_pages--;
  771. h->free_huge_pages_node[nid]--;
  772. update_and_free_page(h, page);
  773. }
  774. spin_unlock(&hugetlb_lock);
  775. }
  776. /*
  777. * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
  778. * make specified memory blocks removable from the system.
  779. * Note that start_pfn should aligned with (minimum) hugepage size.
  780. */
  781. void dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
  782. {
  783. unsigned int order = 8 * sizeof(void *);
  784. unsigned long pfn;
  785. struct hstate *h;
  786. /* Set scan step to minimum hugepage size */
  787. for_each_hstate(h)
  788. if (order > huge_page_order(h))
  789. order = huge_page_order(h);
  790. VM_BUG_ON(!IS_ALIGNED(start_pfn, 1 << order));
  791. for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << order)
  792. dissolve_free_huge_page(pfn_to_page(pfn));
  793. }
  794. static struct page *alloc_buddy_huge_page(struct hstate *h, int nid)
  795. {
  796. struct page *page;
  797. unsigned int r_nid;
  798. if (h->order >= MAX_ORDER)
  799. return NULL;
  800. /*
  801. * Assume we will successfully allocate the surplus page to
  802. * prevent racing processes from causing the surplus to exceed
  803. * overcommit
  804. *
  805. * This however introduces a different race, where a process B
  806. * tries to grow the static hugepage pool while alloc_pages() is
  807. * called by process A. B will only examine the per-node
  808. * counters in determining if surplus huge pages can be
  809. * converted to normal huge pages in adjust_pool_surplus(). A
  810. * won't be able to increment the per-node counter, until the
  811. * lock is dropped by B, but B doesn't drop hugetlb_lock until
  812. * no more huge pages can be converted from surplus to normal
  813. * state (and doesn't try to convert again). Thus, we have a
  814. * case where a surplus huge page exists, the pool is grown, and
  815. * the surplus huge page still exists after, even though it
  816. * should just have been converted to a normal huge page. This
  817. * does not leak memory, though, as the hugepage will be freed
  818. * once it is out of use. It also does not allow the counters to
  819. * go out of whack in adjust_pool_surplus() as we don't modify
  820. * the node values until we've gotten the hugepage and only the
  821. * per-node value is checked there.
  822. */
  823. spin_lock(&hugetlb_lock);
  824. if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
  825. spin_unlock(&hugetlb_lock);
  826. return NULL;
  827. } else {
  828. h->nr_huge_pages++;
  829. h->surplus_huge_pages++;
  830. }
  831. spin_unlock(&hugetlb_lock);
  832. if (nid == NUMA_NO_NODE)
  833. page = alloc_pages(htlb_alloc_mask(h)|__GFP_COMP|
  834. __GFP_REPEAT|__GFP_NOWARN,
  835. huge_page_order(h));
  836. else
  837. page = alloc_pages_exact_node(nid,
  838. htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
  839. __GFP_REPEAT|__GFP_NOWARN, huge_page_order(h));
  840. if (page && arch_prepare_hugepage(page)) {
  841. __free_pages(page, huge_page_order(h));
  842. page = NULL;
  843. }
  844. spin_lock(&hugetlb_lock);
  845. if (page) {
  846. INIT_LIST_HEAD(&page->lru);
  847. r_nid = page_to_nid(page);
  848. set_compound_page_dtor(page, free_huge_page);
  849. set_hugetlb_cgroup(page, NULL);
  850. /*
  851. * We incremented the global counters already
  852. */
  853. h->nr_huge_pages_node[r_nid]++;
  854. h->surplus_huge_pages_node[r_nid]++;
  855. __count_vm_event(HTLB_BUDDY_PGALLOC);
  856. } else {
  857. h->nr_huge_pages--;
  858. h->surplus_huge_pages--;
  859. __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
  860. }
  861. spin_unlock(&hugetlb_lock);
  862. return page;
  863. }
  864. /*
  865. * This allocation function is useful in the context where vma is irrelevant.
  866. * E.g. soft-offlining uses this function because it only cares physical
  867. * address of error page.
  868. */
  869. struct page *alloc_huge_page_node(struct hstate *h, int nid)
  870. {
  871. struct page *page = NULL;
  872. spin_lock(&hugetlb_lock);
  873. if (h->free_huge_pages - h->resv_huge_pages > 0)
  874. page = dequeue_huge_page_node(h, nid);
  875. spin_unlock(&hugetlb_lock);
  876. if (!page)
  877. page = alloc_buddy_huge_page(h, nid);
  878. return page;
  879. }
  880. /*
  881. * Increase the hugetlb pool such that it can accommodate a reservation
  882. * of size 'delta'.
  883. */
  884. static int gather_surplus_pages(struct hstate *h, int delta)
  885. {
  886. struct list_head surplus_list;
  887. struct page *page, *tmp;
  888. int ret, i;
  889. int needed, allocated;
  890. bool alloc_ok = true;
  891. needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
  892. if (needed <= 0) {
  893. h->resv_huge_pages += delta;
  894. return 0;
  895. }
  896. allocated = 0;
  897. INIT_LIST_HEAD(&surplus_list);
  898. ret = -ENOMEM;
  899. retry:
  900. spin_unlock(&hugetlb_lock);
  901. for (i = 0; i < needed; i++) {
  902. page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
  903. if (!page) {
  904. alloc_ok = false;
  905. break;
  906. }
  907. list_add(&page->lru, &surplus_list);
  908. }
  909. allocated += i;
  910. /*
  911. * After retaking hugetlb_lock, we need to recalculate 'needed'
  912. * because either resv_huge_pages or free_huge_pages may have changed.
  913. */
  914. spin_lock(&hugetlb_lock);
  915. needed = (h->resv_huge_pages + delta) -
  916. (h->free_huge_pages + allocated);
  917. if (needed > 0) {
  918. if (alloc_ok)
  919. goto retry;
  920. /*
  921. * We were not able to allocate enough pages to
  922. * satisfy the entire reservation so we free what
  923. * we've allocated so far.
  924. */
  925. goto free;
  926. }
  927. /*
  928. * The surplus_list now contains _at_least_ the number of extra pages
  929. * needed to accommodate the reservation. Add the appropriate number
  930. * of pages to the hugetlb pool and free the extras back to the buddy
  931. * allocator. Commit the entire reservation here to prevent another
  932. * process from stealing the pages as they are added to the pool but
  933. * before they are reserved.
  934. */
  935. needed += allocated;
  936. h->resv_huge_pages += delta;
  937. ret = 0;
  938. /* Free the needed pages to the hugetlb pool */
  939. list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
  940. if ((--needed) < 0)
  941. break;
  942. /*
  943. * This page is now managed by the hugetlb allocator and has
  944. * no users -- drop the buddy allocator's reference.
  945. */
  946. put_page_testzero(page);
  947. VM_BUG_ON(page_count(page));
  948. enqueue_huge_page(h, page);
  949. }
  950. free:
  951. spin_unlock(&hugetlb_lock);
  952. /* Free unnecessary surplus pages to the buddy allocator */
  953. list_for_each_entry_safe(page, tmp, &surplus_list, lru)
  954. put_page(page);
  955. spin_lock(&hugetlb_lock);
  956. return ret;
  957. }
  958. /*
  959. * When releasing a hugetlb pool reservation, any surplus pages that were
  960. * allocated to satisfy the reservation must be explicitly freed if they were
  961. * never used.
  962. * Called with hugetlb_lock held.
  963. */
  964. static void return_unused_surplus_pages(struct hstate *h,
  965. unsigned long unused_resv_pages)
  966. {
  967. unsigned long nr_pages;
  968. /* Uncommit the reservation */
  969. h->resv_huge_pages -= unused_resv_pages;
  970. /* Cannot return gigantic pages currently */
  971. if (h->order >= MAX_ORDER)
  972. return;
  973. nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
  974. /*
  975. * We want to release as many surplus pages as possible, spread
  976. * evenly across all nodes with memory. Iterate across these nodes
  977. * until we can no longer free unreserved surplus pages. This occurs
  978. * when the nodes with surplus pages have no free pages.
  979. * free_pool_huge_page() will balance the the freed pages across the
  980. * on-line nodes with memory and will handle the hstate accounting.
  981. */
  982. while (nr_pages--) {
  983. if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1))
  984. break;
  985. }
  986. }
  987. /*
  988. * Determine if the huge page at addr within the vma has an associated
  989. * reservation. Where it does not we will need to logically increase
  990. * reservation and actually increase subpool usage before an allocation
  991. * can occur. Where any new reservation would be required the
  992. * reservation change is prepared, but not committed. Once the page
  993. * has been allocated from the subpool and instantiated the change should
  994. * be committed via vma_commit_reservation. No action is required on
  995. * failure.
  996. */
  997. static long vma_needs_reservation(struct hstate *h,
  998. struct vm_area_struct *vma, unsigned long addr)
  999. {
  1000. struct address_space *mapping = vma->vm_file->f_mapping;
  1001. struct inode *inode = mapping->host;
  1002. if (vma->vm_flags & VM_MAYSHARE) {
  1003. pgoff_t idx = vma_hugecache_offset(h, vma, addr);
  1004. return region_chg(&inode->i_mapping->private_list,
  1005. idx, idx + 1);
  1006. } else if (!is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
  1007. return 1;
  1008. } else {
  1009. long err;
  1010. pgoff_t idx = vma_hugecache_offset(h, vma, addr);
  1011. struct resv_map *resv = vma_resv_map(vma);
  1012. err = region_chg(&resv->regions, idx, idx + 1);
  1013. if (err < 0)
  1014. return err;
  1015. return 0;
  1016. }
  1017. }
  1018. static void vma_commit_reservation(struct hstate *h,
  1019. struct vm_area_struct *vma, unsigned long addr)
  1020. {
  1021. struct address_space *mapping = vma->vm_file->f_mapping;
  1022. struct inode *inode = mapping->host;
  1023. if (vma->vm_flags & VM_MAYSHARE) {
  1024. pgoff_t idx = vma_hugecache_offset(h, vma, addr);
  1025. region_add(&inode->i_mapping->private_list, idx, idx + 1);
  1026. } else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
  1027. pgoff_t idx = vma_hugecache_offset(h, vma, addr);
  1028. struct resv_map *resv = vma_resv_map(vma);
  1029. /* Mark this page used in the map. */
  1030. region_add(&resv->regions, idx, idx + 1);
  1031. }
  1032. }
  1033. static struct page *alloc_huge_page(struct vm_area_struct *vma,
  1034. unsigned long addr, int avoid_reserve)
  1035. {
  1036. struct hugepage_subpool *spool = subpool_vma(vma);
  1037. struct hstate *h = hstate_vma(vma);
  1038. struct page *page;
  1039. long chg;
  1040. int ret, idx;
  1041. struct hugetlb_cgroup *h_cg;
  1042. idx = hstate_index(h);
  1043. /*
  1044. * Processes that did not create the mapping will have no
  1045. * reserves and will not have accounted against subpool
  1046. * limit. Check that the subpool limit can be made before
  1047. * satisfying the allocation MAP_NORESERVE mappings may also
  1048. * need pages and subpool limit allocated allocated if no reserve
  1049. * mapping overlaps.
  1050. */
  1051. chg = vma_needs_reservation(h, vma, addr);
  1052. if (chg < 0)
  1053. return ERR_PTR(-ENOMEM);
  1054. if (chg || avoid_reserve)
  1055. if (hugepage_subpool_get_pages(spool, 1))
  1056. return ERR_PTR(-ENOSPC);
  1057. ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
  1058. if (ret) {
  1059. if (chg || avoid_reserve)
  1060. hugepage_subpool_put_pages(spool, 1);
  1061. return ERR_PTR(-ENOSPC);
  1062. }
  1063. spin_lock(&hugetlb_lock);
  1064. page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, chg);
  1065. if (!page) {
  1066. spin_unlock(&hugetlb_lock);
  1067. page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
  1068. if (!page) {
  1069. hugetlb_cgroup_uncharge_cgroup(idx,
  1070. pages_per_huge_page(h),
  1071. h_cg);
  1072. if (chg || avoid_reserve)
  1073. hugepage_subpool_put_pages(spool, 1);
  1074. return ERR_PTR(-ENOSPC);
  1075. }
  1076. spin_lock(&hugetlb_lock);
  1077. list_move(&page->lru, &h->hugepage_activelist);
  1078. /* Fall through */
  1079. }
  1080. hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
  1081. spin_unlock(&hugetlb_lock);
  1082. set_page_private(page, (unsigned long)spool);
  1083. vma_commit_reservation(h, vma, addr);
  1084. return page;
  1085. }
  1086. /*
  1087. * alloc_huge_page()'s wrapper which simply returns the page if allocation
  1088. * succeeds, otherwise NULL. This function is called from new_vma_page(),
  1089. * where no ERR_VALUE is expected to be returned.
  1090. */
  1091. struct page *alloc_huge_page_noerr(struct vm_area_struct *vma,
  1092. unsigned long addr, int avoid_reserve)
  1093. {
  1094. struct page *page = alloc_huge_page(vma, addr, avoid_reserve);
  1095. if (IS_ERR(page))
  1096. page = NULL;
  1097. return page;
  1098. }
  1099. int __weak alloc_bootmem_huge_page(struct hstate *h)
  1100. {
  1101. struct huge_bootmem_page *m;
  1102. int nr_nodes, node;
  1103. for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
  1104. void *addr;
  1105. addr = __alloc_bootmem_node_nopanic(NODE_DATA(node),
  1106. huge_page_size(h), huge_page_size(h), 0);
  1107. if (addr) {
  1108. /*
  1109. * Use the beginning of the huge page to store the
  1110. * huge_bootmem_page struct (until gather_bootmem
  1111. * puts them into the mem_map).
  1112. */
  1113. m = addr;
  1114. goto found;
  1115. }
  1116. }
  1117. return 0;
  1118. found:
  1119. BUG_ON((unsigned long)virt_to_phys(m) & (huge_page_size(h) - 1));
  1120. /* Put them into a private list first because mem_map is not up yet */
  1121. list_add(&m->list, &huge_boot_pages);
  1122. m->hstate = h;
  1123. return 1;
  1124. }
  1125. static void prep_compound_huge_page(struct page *page, int order)
  1126. {
  1127. if (unlikely(order > (MAX_ORDER - 1)))
  1128. prep_compound_gigantic_page(page, order);
  1129. else
  1130. prep_compound_page(page, order);
  1131. }
  1132. /* Put bootmem huge pages into the standard lists after mem_map is up */
  1133. static void __init gather_bootmem_prealloc(void)
  1134. {
  1135. struct huge_bootmem_page *m;
  1136. list_for_each_entry(m, &huge_boot_pages, list) {
  1137. struct hstate *h = m->hstate;
  1138. struct page *page;
  1139. #ifdef CONFIG_HIGHMEM
  1140. page = pfn_to_page(m->phys >> PAGE_SHIFT);
  1141. free_bootmem_late((unsigned long)m,
  1142. sizeof(struct huge_bootmem_page));
  1143. #else
  1144. page = virt_to_page(m);
  1145. #endif
  1146. WARN_ON(page_count(page) != 1);
  1147. prep_compound_huge_page(page, h->order);
  1148. WARN_ON(PageReserved(page));
  1149. prep_new_huge_page(h, page, page_to_nid(page));
  1150. /*
  1151. * If we had gigantic hugepages allocated at boot time, we need
  1152. * to restore the 'stolen' pages to totalram_pages in order to
  1153. * fix confusing memory reports from free(1) and another
  1154. * side-effects, like CommitLimit going negative.
  1155. */
  1156. if (h->order > (MAX_ORDER - 1))
  1157. adjust_managed_page_count(page, 1 << h->order);
  1158. }
  1159. }
  1160. static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
  1161. {
  1162. unsigned long i;
  1163. for (i = 0; i < h->max_huge_pages; ++i) {
  1164. if (h->order >= MAX_ORDER) {
  1165. if (!alloc_bootmem_huge_page(h))
  1166. break;
  1167. } else if (!alloc_fresh_huge_page(h,
  1168. &node_states[N_MEMORY]))
  1169. break;
  1170. }
  1171. h->max_huge_pages = i;
  1172. }
  1173. static void __init hugetlb_init_hstates(void)
  1174. {
  1175. struct hstate *h;
  1176. for_each_hstate(h) {
  1177. /* oversize hugepages were init'ed in early boot */
  1178. if (h->order < MAX_ORDER)
  1179. hugetlb_hstate_alloc_pages(h);
  1180. }
  1181. }
  1182. static char * __init memfmt(char *buf, unsigned long n)
  1183. {
  1184. if (n >= (1UL << 30))
  1185. sprintf(buf, "%lu GB", n >> 30);
  1186. else if (n >= (1UL << 20))
  1187. sprintf(buf, "%lu MB", n >> 20);
  1188. else
  1189. sprintf(buf, "%lu KB", n >> 10);
  1190. return buf;
  1191. }
  1192. static void __init report_hugepages(void)
  1193. {
  1194. struct hstate *h;
  1195. for_each_hstate(h) {
  1196. char buf[32];
  1197. pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n",
  1198. memfmt(buf, huge_page_size(h)),
  1199. h->free_huge_pages);
  1200. }
  1201. }
  1202. #ifdef CONFIG_HIGHMEM
  1203. static void try_to_free_low(struct hstate *h, unsigned long count,
  1204. nodemask_t *nodes_allowed)
  1205. {
  1206. int i;
  1207. if (h->order >= MAX_ORDER)
  1208. return;
  1209. for_each_node_mask(i, *nodes_allowed) {
  1210. struct page *page, *next;
  1211. struct list_head *freel = &h->hugepage_freelists[i];
  1212. list_for_each_entry_safe(page, next, freel, lru) {
  1213. if (count >= h->nr_huge_pages)
  1214. return;
  1215. if (PageHighMem(page))
  1216. continue;
  1217. list_del(&page->lru);
  1218. update_and_free_page(h, page);
  1219. h->free_huge_pages--;
  1220. h->free_huge_pages_node[page_to_nid(page)]--;
  1221. }
  1222. }
  1223. }
  1224. #else
  1225. static inline void try_to_free_low(struct hstate *h, unsigned long count,
  1226. nodemask_t *nodes_allowed)
  1227. {
  1228. }
  1229. #endif
  1230. /*
  1231. * Increment or decrement surplus_huge_pages. Keep node-specific counters
  1232. * balanced by operating on them in a round-robin fashion.
  1233. * Returns 1 if an adjustment was made.
  1234. */
  1235. static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
  1236. int delta)
  1237. {
  1238. int nr_nodes, node;
  1239. VM_BUG_ON(delta != -1 && delta != 1);
  1240. if (delta < 0) {
  1241. for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
  1242. if (h->surplus_huge_pages_node[node])
  1243. goto found;
  1244. }
  1245. } else {
  1246. for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
  1247. if (h->surplus_huge_pages_node[node] <
  1248. h->nr_huge_pages_node[node])
  1249. goto found;
  1250. }
  1251. }
  1252. return 0;
  1253. found:
  1254. h->surplus_huge_pages += delta;
  1255. h->surplus_huge_pages_node[node] += delta;
  1256. return 1;
  1257. }
  1258. #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
  1259. static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
  1260. nodemask_t *nodes_allowed)
  1261. {
  1262. unsigned long min_count, ret;
  1263. if (h->order >= MAX_ORDER)
  1264. return h->max_huge_pages;
  1265. /*
  1266. * Increase the pool size
  1267. * First take pages out of surplus state. Then make up the
  1268. * remaining difference by allocating fresh huge pages.
  1269. *
  1270. * We might race with alloc_buddy_huge_page() here and be unable
  1271. * to convert a surplus huge page to a normal huge page. That is
  1272. * not critical, though, it just means the overall size of the
  1273. * pool might be one hugepage larger than it needs to be, but
  1274. * within all the constraints specified by the sysctls.
  1275. */
  1276. spin_lock(&hugetlb_lock);
  1277. while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
  1278. if (!adjust_pool_surplus(h, nodes_allowed, -1))
  1279. break;
  1280. }
  1281. while (count > persistent_huge_pages(h)) {
  1282. /*
  1283. * If this allocation races such that we no longer need the
  1284. * page, free_huge_page will handle it by freeing the page
  1285. * and reducing the surplus.
  1286. */
  1287. spin_unlock(&hugetlb_lock);
  1288. ret = alloc_fresh_huge_page(h, nodes_allowed);
  1289. spin_lock(&hugetlb_lock);
  1290. if (!ret)
  1291. goto out;
  1292. /* Bail for signals. Probably ctrl-c from user */
  1293. if (signal_pending(current))
  1294. goto out;
  1295. }
  1296. /*
  1297. * Decrease the pool size
  1298. * First return free pages to the buddy allocator (being careful
  1299. * to keep enough around to satisfy reservations). Then place
  1300. * pages into surplus state as needed so the pool will shrink
  1301. * to the desired size as pages become free.
  1302. *
  1303. * By placing pages into the surplus state independent of the
  1304. * overcommit value, we are allowing the surplus pool size to
  1305. * exceed overcommit. There are few sane options here. Since
  1306. * alloc_buddy_huge_page() is checking the global counter,
  1307. * though, we'll note that we're not allowed to exceed surplus
  1308. * and won't grow the pool anywhere else. Not until one of the
  1309. * sysctls are changed, or the surplus pages go out of use.
  1310. */
  1311. min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
  1312. min_count = max(count, min_count);
  1313. try_to_free_low(h, min_count, nodes_allowed);
  1314. while (min_count < persistent_huge_pages(h)) {
  1315. if (!free_pool_huge_page(h, nodes_allowed, 0))
  1316. break;
  1317. }
  1318. while (count < persistent_huge_pages(h)) {
  1319. if (!adjust_pool_surplus(h, nodes_allowed, 1))
  1320. break;
  1321. }
  1322. out:
  1323. ret = persistent_huge_pages(h);
  1324. spin_unlock(&hugetlb_lock);
  1325. return ret;
  1326. }
  1327. #define HSTATE_ATTR_RO(_name) \
  1328. static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
  1329. #define HSTATE_ATTR(_name) \
  1330. static struct kobj_attribute _name##_attr = \
  1331. __ATTR(_name, 0644, _name##_show, _name##_store)
  1332. static struct kobject *hugepages_kobj;
  1333. static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
  1334. static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
  1335. static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
  1336. {
  1337. int i;
  1338. for (i = 0; i < HUGE_MAX_HSTATE; i++)
  1339. if (hstate_kobjs[i] == kobj) {
  1340. if (nidp)
  1341. *nidp = NUMA_NO_NODE;
  1342. return &hstates[i];
  1343. }
  1344. return kobj_to_node_hstate(kobj, nidp);
  1345. }
  1346. static ssize_t nr_hugepages_show_common(struct kobject *kobj,
  1347. struct kobj_attribute *attr, char *buf)
  1348. {
  1349. struct hstate *h;
  1350. unsigned long nr_huge_pages;
  1351. int nid;
  1352. h = kobj_to_hstate(kobj, &nid);
  1353. if (nid == NUMA_NO_NODE)
  1354. nr_huge_pages = h->nr_huge_pages;
  1355. else
  1356. nr_huge_pages = h->nr_huge_pages_node[nid];
  1357. return sprintf(buf, "%lu\n", nr_huge_pages);
  1358. }
  1359. static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
  1360. struct kobject *kobj, struct kobj_attribute *attr,
  1361. const char *buf, size_t len)
  1362. {
  1363. int err;
  1364. int nid;
  1365. unsigned long count;
  1366. struct hstate *h;
  1367. NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY);
  1368. err = kstrtoul(buf, 10, &count);
  1369. if (err)
  1370. goto out;
  1371. h = kobj_to_hstate(kobj, &nid);
  1372. if (h->order >= MAX_ORDER) {
  1373. err = -EINVAL;
  1374. goto out;
  1375. }
  1376. if (nid == NUMA_NO_NODE) {
  1377. /*
  1378. * global hstate attribute
  1379. */
  1380. if (!(obey_mempolicy &&
  1381. init_nodemask_of_mempolicy(nodes_allowed))) {
  1382. NODEMASK_FREE(nodes_allowed);
  1383. nodes_allowed = &node_states[N_MEMORY];
  1384. }
  1385. } else if (nodes_allowed) {
  1386. /*
  1387. * per node hstate attribute: adjust count to global,
  1388. * but restrict alloc/free to the specified node.
  1389. */
  1390. count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
  1391. init_nodemask_of_node(nodes_allowed, nid);
  1392. } else
  1393. nodes_allowed = &node_states[N_MEMORY];
  1394. h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
  1395. if (nodes_allowed != &node_states[N_MEMORY])
  1396. NODEMASK_FREE(nodes_allowed);
  1397. return len;
  1398. out:
  1399. NODEMASK_FREE(nodes_allowed);
  1400. return err;
  1401. }
  1402. static ssize_t nr_hugepages_show(struct kobject *kobj,
  1403. struct kobj_attribute *attr, char *buf)
  1404. {
  1405. return nr_hugepages_show_common(kobj, attr, buf);
  1406. }
  1407. static ssize_t nr_hugepages_store(struct kobject *kobj,
  1408. struct kobj_attribute *attr, const char *buf, size_t len)
  1409. {
  1410. return nr_hugepages_store_common(false, kobj, attr, buf, len);
  1411. }
  1412. HSTATE_ATTR(nr_hugepages);
  1413. #ifdef CONFIG_NUMA
  1414. /*
  1415. * hstate attribute for optionally mempolicy-based constraint on persistent
  1416. * huge page alloc/free.
  1417. */
  1418. static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
  1419. struct kobj_attribute *attr, char *buf)
  1420. {
  1421. return nr_hugepages_show_common(kobj, attr, buf);
  1422. }
  1423. static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
  1424. struct kobj_attribute *attr, const char *buf, size_t len)
  1425. {
  1426. return nr_hugepages_store_common(true, kobj, attr, buf, len);
  1427. }
  1428. HSTATE_ATTR(nr_hugepages_mempolicy);
  1429. #endif
  1430. static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
  1431. struct kobj_attribute *attr, char *buf)
  1432. {
  1433. struct hstate *h = kobj_to_hstate(kobj, NULL);
  1434. return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
  1435. }
  1436. static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
  1437. struct kobj_attribute *attr, const char *buf, size_t count)
  1438. {
  1439. int err;
  1440. unsigned long input;
  1441. struct hstate *h = kobj_to_hstate(kobj, NULL);
  1442. if (h->order >= MAX_ORDER)
  1443. return -EINVAL;
  1444. err = kstrtoul(buf, 10, &input);
  1445. if (err)
  1446. return err;
  1447. spin_lock(&hugetlb_lock);
  1448. h->nr_overcommit_huge_pages = input;
  1449. spin_unlock(&hugetlb_lock);
  1450. return count;
  1451. }
  1452. HSTATE_ATTR(nr_overcommit_hugepages);
  1453. static ssize_t free_hugepages_show(struct kobject *kobj,
  1454. struct kobj_attribute *attr, char *buf)
  1455. {
  1456. struct hstate *h;
  1457. unsigned long free_huge_pages;
  1458. int nid;
  1459. h = kobj_to_hstate(kobj, &nid);
  1460. if (nid == NUMA_NO_NODE)
  1461. free_huge_pages = h->free_huge_pages;
  1462. else
  1463. free_huge_pages = h->free_huge_pages_node[nid];
  1464. return sprintf(buf, "%lu\n", free_huge_pages);
  1465. }
  1466. HSTATE_ATTR_RO(free_hugepages);
  1467. static ssize_t resv_hugepages_show(struct kobject *kobj,
  1468. struct kobj_attribute *attr, char *buf)
  1469. {
  1470. struct hstate *h = kobj_to_hstate(kobj, NULL);
  1471. return sprintf(buf, "%lu\n", h->resv_huge_pages);
  1472. }
  1473. HSTATE_ATTR_RO(resv_hugepages);
  1474. static ssize_t surplus_hugepages_show(struct kobject *kobj,
  1475. struct kobj_attribute *attr, char *buf)
  1476. {
  1477. struct hstate *h;
  1478. unsigned long surplus_huge_pages;
  1479. int nid;
  1480. h = kobj_to_hstate(kobj, &nid);
  1481. if (nid == NUMA_NO_NODE)
  1482. surplus_huge_pages = h->surplus_huge_pages;
  1483. else
  1484. surplus_huge_pages = h->surplus_huge_pages_node[nid];
  1485. return sprintf(buf, "%lu\n", surplus_huge_pages);
  1486. }
  1487. HSTATE_ATTR_RO(surplus_hugepages);
  1488. static struct attribute *hstate_attrs[] = {
  1489. &nr_hugepages_attr.attr,
  1490. &nr_overcommit_hugepages_attr.attr,
  1491. &free_hugepages_attr.attr,
  1492. &resv_hugepages_attr.attr,
  1493. &surplus_hugepages_attr.attr,
  1494. #ifdef CONFIG_NUMA
  1495. &nr_hugepages_mempolicy_attr.attr,
  1496. #endif
  1497. NULL,
  1498. };
  1499. static struct attribute_group hstate_attr_group = {
  1500. .attrs = hstate_attrs,
  1501. };
  1502. static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
  1503. struct kobject **hstate_kobjs,
  1504. struct attribute_group *hstate_attr_group)
  1505. {
  1506. int retval;
  1507. int hi = hstate_index(h);
  1508. hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
  1509. if (!hstate_kobjs[hi])
  1510. return -ENOMEM;
  1511. retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
  1512. if (retval)
  1513. kobject_put(hstate_kobjs[hi]);
  1514. return retval;
  1515. }
  1516. static void __init hugetlb_sysfs_init(void)
  1517. {
  1518. struct hstate *h;
  1519. int err;
  1520. hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
  1521. if (!hugepages_kobj)
  1522. return;
  1523. for_each_hstate(h) {
  1524. err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
  1525. hstate_kobjs, &hstate_attr_group);
  1526. if (err)
  1527. pr_err("Hugetlb: Unable to add hstate %s", h->name);
  1528. }
  1529. }
  1530. #ifdef CONFIG_NUMA
  1531. /*
  1532. * node_hstate/s - associate per node hstate attributes, via their kobjects,
  1533. * with node devices in node_devices[] using a parallel array. The array
  1534. * index of a node device or _hstate == node id.
  1535. * This is here to avoid any static dependency of the node device driver, in
  1536. * the base kernel, on the hugetlb module.
  1537. */
  1538. struct node_hstate {
  1539. struct kobject *hugepages_kobj;
  1540. struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
  1541. };
  1542. struct node_hstate node_hstates[MAX_NUMNODES];
  1543. /*
  1544. * A subset of global hstate attributes for node devices
  1545. */
  1546. static struct attribute *per_node_hstate_attrs[] = {
  1547. &nr_hugepages_attr.attr,
  1548. &free_hugepages_attr.attr,
  1549. &surplus_hugepages_attr.attr,
  1550. NULL,
  1551. };
  1552. static struct attribute_group per_node_hstate_attr_group = {
  1553. .attrs = per_node_hstate_attrs,
  1554. };
  1555. /*
  1556. * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
  1557. * Returns node id via non-NULL nidp.
  1558. */
  1559. static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
  1560. {
  1561. int nid;
  1562. for (nid = 0; nid < nr_node_ids; nid++) {
  1563. struct node_hstate *nhs = &node_hstates[nid];
  1564. int i;
  1565. for (i = 0; i < HUGE_MAX_HSTATE; i++)
  1566. if (nhs->hstate_kobjs[i] == kobj) {
  1567. if (nidp)
  1568. *nidp = nid;
  1569. return &hstates[i];
  1570. }
  1571. }
  1572. BUG();
  1573. return NULL;
  1574. }
  1575. /*
  1576. * Unregister hstate attributes from a single node device.
  1577. * No-op if no hstate attributes attached.
  1578. */
  1579. static void hugetlb_unregister_node(struct node *node)
  1580. {
  1581. struct hstate *h;
  1582. struct node_hstate *nhs = &node_hstates[node->dev.id];
  1583. if (!nhs->hugepages_kobj)
  1584. return; /* no hstate attributes */
  1585. for_each_hstate(h) {
  1586. int idx = hstate_index(h);
  1587. if (nhs->hstate_kobjs[idx]) {
  1588. kobject_put(nhs->hstate_kobjs[idx]);
  1589. nhs->hstate_kobjs[idx] = NULL;
  1590. }
  1591. }
  1592. kobject_put(nhs->hugepages_kobj);
  1593. nhs->hugepages_kobj = NULL;
  1594. }
  1595. /*
  1596. * hugetlb module exit: unregister hstate attributes from node devices
  1597. * that have them.
  1598. */
  1599. static void hugetlb_unregister_all_nodes(void)
  1600. {
  1601. int nid;
  1602. /*
  1603. * disable node device registrations.
  1604. */
  1605. register_hugetlbfs_with_node(NULL, NULL);
  1606. /*
  1607. * remove hstate attributes from any nodes that have them.
  1608. */
  1609. for (nid = 0; nid < nr_node_ids; nid++)
  1610. hugetlb_unregister_node(node_devices[nid]);
  1611. }
  1612. /*
  1613. * Register hstate attributes for a single node device.
  1614. * No-op if attributes already registered.
  1615. */
  1616. static void hugetlb_register_node(struct node *node)
  1617. {
  1618. struct hstate *h;
  1619. struct node_hstate *nhs = &node_hstates[node->dev.id];
  1620. int err;
  1621. if (nhs->hugepages_kobj)
  1622. return; /* already allocated */
  1623. nhs->hugepages_kobj = kobject_create_and_add("hugepages",
  1624. &node->dev.kobj);
  1625. if (!nhs->hugepages_kobj)
  1626. return;
  1627. for_each_hstate(h) {
  1628. err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
  1629. nhs->hstate_kobjs,
  1630. &per_node_hstate_attr_group);
  1631. if (err) {
  1632. pr_err("Hugetlb: Unable to add hstate %s for node %d\n",
  1633. h->name, node->dev.id);
  1634. hugetlb_unregister_node(node);
  1635. break;
  1636. }
  1637. }
  1638. }
  1639. /*
  1640. * hugetlb init time: register hstate attributes for all registered node
  1641. * devices of nodes that have memory. All on-line nodes should have
  1642. * registered their associated device by this time.
  1643. */
  1644. static void hugetlb_register_all_nodes(void)
  1645. {
  1646. int nid;
  1647. for_each_node_state(nid, N_MEMORY) {
  1648. struct node *node = node_devices[nid];
  1649. if (node->dev.id == nid)
  1650. hugetlb_register_node(node);
  1651. }
  1652. /*
  1653. * Let the node device driver know we're here so it can
  1654. * [un]register hstate attributes on node hotplug.
  1655. */
  1656. register_hugetlbfs_with_node(hugetlb_register_node,
  1657. hugetlb_unregister_node);
  1658. }
  1659. #else /* !CONFIG_NUMA */
  1660. static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
  1661. {
  1662. BUG();
  1663. if (nidp)
  1664. *nidp = -1;
  1665. return NULL;
  1666. }
  1667. static void hugetlb_unregister_all_nodes(void) { }
  1668. static void hugetlb_register_all_nodes(void) { }
  1669. #endif
  1670. static void __exit hugetlb_exit(void)
  1671. {
  1672. struct hstate *h;
  1673. hugetlb_unregister_all_nodes();
  1674. for_each_hstate(h) {
  1675. kobject_put(hstate_kobjs[hstate_index(h)]);
  1676. }
  1677. kobject_put(hugepages_kobj);
  1678. }
  1679. module_exit(hugetlb_exit);
  1680. static int __init hugetlb_init(void)
  1681. {
  1682. /* Some platform decide whether they support huge pages at boot
  1683. * time. On these, such as powerpc, HPAGE_SHIFT is set to 0 when
  1684. * there is no such support
  1685. */
  1686. if (HPAGE_SHIFT == 0)
  1687. return 0;
  1688. if (!size_to_hstate(default_hstate_size)) {
  1689. default_hstate_size = HPAGE_SIZE;
  1690. if (!size_to_hstate(default_hstate_size))
  1691. hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
  1692. }
  1693. default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size));
  1694. if (default_hstate_max_huge_pages)
  1695. default_hstate.max_huge_pages = default_hstate_max_huge_pages;
  1696. hugetlb_init_hstates();
  1697. gather_bootmem_prealloc();
  1698. report_hugepages();
  1699. hugetlb_sysfs_init();
  1700. hugetlb_register_all_nodes();
  1701. hugetlb_cgroup_file_init();
  1702. return 0;
  1703. }
  1704. module_init(hugetlb_init);
  1705. /* Should be called on processing a hugepagesz=... option */
  1706. void __init hugetlb_add_hstate(unsigned order)
  1707. {
  1708. struct hstate *h;
  1709. unsigned long i;
  1710. if (size_to_hstate(PAGE_SIZE << order)) {
  1711. pr_warning("hugepagesz= specified twice, ignoring\n");
  1712. return;
  1713. }
  1714. BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
  1715. BUG_ON(order == 0);
  1716. h = &hstates[hugetlb_max_hstate++];
  1717. h->order = order;
  1718. h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
  1719. h->nr_huge_pages = 0;
  1720. h->free_huge_pages = 0;
  1721. for (i = 0; i < MAX_NUMNODES; ++i)
  1722. INIT_LIST_HEAD(&h->hugepage_freelists[i]);
  1723. INIT_LIST_HEAD(&h->hugepage_activelist);
  1724. h->next_nid_to_alloc = first_node(node_states[N_MEMORY]);
  1725. h->next_nid_to_free = first_node(node_states[N_MEMORY]);
  1726. snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
  1727. huge_page_size(h)/1024);
  1728. parsed_hstate = h;
  1729. }
  1730. static int __init hugetlb_nrpages_setup(char *s)
  1731. {
  1732. unsigned long *mhp;
  1733. static unsigned long *last_mhp;
  1734. /*
  1735. * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter yet,
  1736. * so this hugepages= parameter goes to the "default hstate".
  1737. */
  1738. if (!hugetlb_max_hstate)
  1739. mhp = &default_hstate_max_huge_pages;
  1740. else
  1741. mhp = &parsed_hstate->max_huge_pages;
  1742. if (mhp == last_mhp) {
  1743. pr_warning("hugepages= specified twice without "
  1744. "interleaving hugepagesz=, ignoring\n");
  1745. return 1;
  1746. }
  1747. if (sscanf(s, "%lu", mhp) <= 0)
  1748. *mhp = 0;
  1749. /*
  1750. * Global state is always initialized later in hugetlb_init.
  1751. * But we need to allocate >= MAX_ORDER hstates here early to still
  1752. * use the bootmem allocator.
  1753. */
  1754. if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER)
  1755. hugetlb_hstate_alloc_pages(parsed_hstate);
  1756. last_mhp = mhp;
  1757. return 1;
  1758. }
  1759. __setup("hugepages=", hugetlb_nrpages_setup);
  1760. static int __init hugetlb_default_setup(char *s)
  1761. {
  1762. default_hstate_size = memparse(s, &s);
  1763. return 1;
  1764. }
  1765. __setup("default_hugepagesz=", hugetlb_default_setup);
  1766. static unsigned int cpuset_mems_nr(unsigned int *array)
  1767. {
  1768. int node;
  1769. unsigned int nr = 0;
  1770. for_each_node_mask(node, cpuset_current_mems_allowed)
  1771. nr += array[node];
  1772. return nr;
  1773. }
  1774. #ifdef CONFIG_SYSCTL
  1775. static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
  1776. struct ctl_table *table, int write,
  1777. void __user *buffer, size_t *length, loff_t *ppos)
  1778. {
  1779. struct hstate *h = &default_hstate;
  1780. unsigned long tmp;
  1781. int ret;
  1782. tmp = h->max_huge_pages;
  1783. if (write && h->order >= MAX_ORDER)
  1784. return -EINVAL;
  1785. table->data = &tmp;
  1786. table->maxlen = sizeof(unsigned long);
  1787. ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
  1788. if (ret)
  1789. goto out;
  1790. if (write) {
  1791. NODEMASK_ALLOC(nodemask_t, nodes_allowed,
  1792. GFP_KERNEL | __GFP_NORETRY);
  1793. if (!(obey_mempolicy &&
  1794. init_nodemask_of_mempolicy(nodes_allowed))) {
  1795. NODEMASK_FREE(nodes_allowed);
  1796. nodes_allowed = &node_states[N_MEMORY];
  1797. }
  1798. h->max_huge_pages = set_max_huge_pages(h, tmp, nodes_allowed);
  1799. if (nodes_allowed != &node_states[N_MEMORY])
  1800. NODEMASK_FREE(nodes_allowed);
  1801. }
  1802. out:
  1803. return ret;
  1804. }
  1805. int hugetlb_sysctl_handler(struct ctl_table *table, int write,
  1806. void __user *buffer, size_t *length, loff_t *ppos)
  1807. {
  1808. return hugetlb_sysctl_handler_common(false, table, write,
  1809. buffer, length, ppos);
  1810. }
  1811. #ifdef CONFIG_NUMA
  1812. int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
  1813. void __user *buffer, size_t *length, loff_t *ppos)
  1814. {
  1815. return hugetlb_sysctl_handler_common(true, table, write,
  1816. buffer, length, ppos);
  1817. }
  1818. #endif /* CONFIG_NUMA */
  1819. int hugetlb_overcommit_handler(struct ctl_table *table, int write,
  1820. void __user *buffer,
  1821. size_t *length, loff_t *ppos)
  1822. {
  1823. struct hstate *h = &default_hstate;
  1824. unsigned long tmp;
  1825. int ret;
  1826. tmp = h->nr_overcommit_huge_pages;
  1827. if (write && h->order >= MAX_ORDER)
  1828. return -EINVAL;
  1829. table->data = &tmp;
  1830. table->maxlen = sizeof(unsigned long);
  1831. ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
  1832. if (ret)
  1833. goto out;
  1834. if (write) {
  1835. spin_lock(&hugetlb_lock);
  1836. h->nr_overcommit_huge_pages = tmp;
  1837. spin_unlock(&hugetlb_lock);
  1838. }
  1839. out:
  1840. return ret;
  1841. }
  1842. #endif /* CONFIG_SYSCTL */
  1843. void hugetlb_report_meminfo(struct seq_file *m)
  1844. {
  1845. struct hstate *h = &default_hstate;
  1846. seq_printf(m,
  1847. "HugePages_Total: %5lu\n"
  1848. "HugePages_Free: %5lu\n"
  1849. "HugePages_Rsvd: %5lu\n"
  1850. "HugePages_Surp: %5lu\n"
  1851. "Hugepagesize: %8lu kB\n",
  1852. h->nr_huge_pages,
  1853. h->free_huge_pages,
  1854. h->resv_huge_pages,
  1855. h->surplus_huge_pages,
  1856. 1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
  1857. }
  1858. int hugetlb_report_node_meminfo(int nid, char *buf)
  1859. {
  1860. struct hstate *h = &default_hstate;
  1861. return sprintf(buf,
  1862. "Node %d HugePages_Total: %5u\n"
  1863. "Node %d HugePages_Free: %5u\n"
  1864. "Node %d HugePages_Surp: %5u\n",
  1865. nid, h->nr_huge_pages_node[nid],
  1866. nid, h->free_huge_pages_node[nid],
  1867. nid, h->surplus_huge_pages_node[nid]);
  1868. }
  1869. void hugetlb_show_meminfo(void)
  1870. {
  1871. struct hstate *h;
  1872. int nid;
  1873. for_each_node_state(nid, N_MEMORY)
  1874. for_each_hstate(h)
  1875. pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
  1876. nid,
  1877. h->nr_huge_pages_node[nid],
  1878. h->free_huge_pages_node[nid],
  1879. h->surplus_huge_pages_node[nid],
  1880. 1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
  1881. }
  1882. /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
  1883. unsigned long hugetlb_total_pages(void)
  1884. {
  1885. struct hstate *h;
  1886. unsigned long nr_total_pages = 0;
  1887. for_each_hstate(h)
  1888. nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
  1889. return nr_total_pages;
  1890. }
  1891. static int hugetlb_acct_memory(struct hstate *h, long delta)
  1892. {
  1893. int ret = -ENOMEM;
  1894. spin_lock(&hugetlb_lock);
  1895. /*
  1896. * When cpuset is configured, it breaks the strict hugetlb page
  1897. * reservation as the accounting is done on a global variable. Such
  1898. * reservation is completely rubbish in the presence of cpuset because
  1899. * the reservation is not checked against page availability for the
  1900. * current cpuset. Application can still potentially OOM'ed by kernel
  1901. * with lack of free htlb page in cpuset that the task is in.
  1902. * Attempt to enforce strict accounting with cpuset is almost
  1903. * impossible (or too ugly) because cpuset is too fluid that
  1904. * task or memory node can be dynamically moved between cpusets.
  1905. *
  1906. * The change of semantics for shared hugetlb mapping with cpuset is
  1907. * undesirable. However, in order to preserve some of the semantics,
  1908. * we fall back to check against current free page availability as
  1909. * a best attempt and hopefully to minimize the impact of changing
  1910. * semantics that cpuset has.
  1911. */
  1912. if (delta > 0) {
  1913. if (gather_surplus_pages(h, delta) < 0)
  1914. goto out;
  1915. if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
  1916. return_unused_surplus_pages(h, delta);
  1917. goto out;
  1918. }
  1919. }
  1920. ret = 0;
  1921. if (delta < 0)
  1922. return_unused_surplus_pages(h, (unsigned long) -delta);
  1923. out:
  1924. spin_unlock(&hugetlb_lock);
  1925. return ret;
  1926. }
  1927. static void hugetlb_vm_op_open(struct vm_area_struct *vma)
  1928. {
  1929. struct resv_map *resv = vma_resv_map(vma);
  1930. /*
  1931. * This new VMA should share its siblings reservation map if present.
  1932. * The VMA will only ever have a valid reservation map pointer where
  1933. * it is being copied for another still existing VMA. As that VMA
  1934. * has a reference to the reservation map it cannot disappear until
  1935. * after this open call completes. It is therefore safe to take a
  1936. * new reference here without additional locking.
  1937. */
  1938. if (resv)
  1939. kref_get(&resv->refs);
  1940. }
  1941. static void resv_map_put(struct vm_area_struct *vma)
  1942. {
  1943. struct resv_map *resv = vma_resv_map(vma);
  1944. if (!resv)
  1945. return;
  1946. kref_put(&resv->refs, resv_map_release);
  1947. }
  1948. static void hugetlb_vm_op_close(struct vm_area_struct *vma)
  1949. {
  1950. struct hstate *h = hstate_vma(vma);
  1951. struct resv_map *resv = vma_resv_map(vma);
  1952. struct hugepage_subpool *spool = subpool_vma(vma);
  1953. unsigned long reserve;
  1954. unsigned long start;
  1955. unsigned long end;
  1956. if (resv) {
  1957. start = vma_hugecache_offset(h, vma, vma->vm_start);
  1958. end = vma_hugecache_offset(h, vma, vma->vm_end);
  1959. reserve = (end - start) -
  1960. region_count(&resv->regions, start, end);
  1961. resv_map_put(vma);
  1962. if (reserve) {
  1963. hugetlb_acct_memory(h, -reserve);
  1964. hugepage_subpool_put_pages(spool, reserve);
  1965. }
  1966. }
  1967. }
  1968. /*
  1969. * We cannot handle pagefaults against hugetlb pages at all. They cause
  1970. * handle_mm_fault() to try to instantiate regular-sized pages in the
  1971. * hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get
  1972. * this far.
  1973. */
  1974. static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  1975. {
  1976. BUG();
  1977. return 0;
  1978. }
  1979. const struct vm_operations_struct hugetlb_vm_ops = {
  1980. .fault = hugetlb_vm_op_fault,
  1981. .open = hugetlb_vm_op_open,
  1982. .close = hugetlb_vm_op_close,
  1983. };
  1984. static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
  1985. int writable)
  1986. {
  1987. pte_t entry;
  1988. if (writable) {
  1989. entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
  1990. vma->vm_page_prot)));
  1991. } else {
  1992. entry = huge_pte_wrprotect(mk_huge_pte(page,
  1993. vma->vm_page_prot));
  1994. }
  1995. entry = pte_mkyoung(entry);
  1996. entry = pte_mkhuge(entry);
  1997. entry = arch_make_huge_pte(entry, vma, page, writable);
  1998. return entry;
  1999. }
  2000. static void set_huge_ptep_writable(struct vm_area_struct *vma,
  2001. unsigned long address, pte_t *ptep)
  2002. {
  2003. pte_t entry;
  2004. entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
  2005. if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
  2006. update_mmu_cache(vma, address, ptep);
  2007. }
  2008. int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
  2009. struct vm_area_struct *vma)
  2010. {
  2011. pte_t *src_pte, *dst_pte, entry;
  2012. struct page *ptepage;
  2013. unsigned long addr;
  2014. int cow;
  2015. struct hstate *h = hstate_vma(vma);
  2016. unsigned long sz = huge_page_size(h);
  2017. cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
  2018. for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
  2019. spinlock_t *src_ptl, *dst_ptl;
  2020. src_pte = huge_pte_offset(src, addr);
  2021. if (!src_pte)
  2022. continue;
  2023. dst_pte = huge_pte_alloc(dst, addr, sz);
  2024. if (!dst_pte)
  2025. goto nomem;
  2026. /* If the pagetables are shared don't copy or take references */
  2027. if (dst_pte == src_pte)
  2028. continue;
  2029. dst_ptl = huge_pte_lock(h, dst, dst_pte);
  2030. src_ptl = huge_pte_lockptr(h, src, src_pte);
  2031. spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
  2032. if (!huge_pte_none(huge_ptep_get(src_pte))) {
  2033. if (cow)
  2034. huge_ptep_set_wrprotect(src, addr, src_pte);
  2035. entry = huge_ptep_get(src_pte);
  2036. ptepage = pte_page(entry);
  2037. get_page(ptepage);
  2038. page_dup_rmap(ptepage);
  2039. set_huge_pte_at(dst, addr, dst_pte, entry);
  2040. }
  2041. spin_unlock(src_ptl);
  2042. spin_unlock(dst_ptl);
  2043. }
  2044. return 0;
  2045. nomem:
  2046. return -ENOMEM;
  2047. }
  2048. static int is_hugetlb_entry_migration(pte_t pte)
  2049. {
  2050. swp_entry_t swp;
  2051. if (huge_pte_none(pte) || pte_present(pte))
  2052. return 0;
  2053. swp = pte_to_swp_entry(pte);
  2054. if (non_swap_entry(swp) && is_migration_entry(swp))
  2055. return 1;
  2056. else
  2057. return 0;
  2058. }
  2059. static int is_hugetlb_entry_hwpoisoned(pte_t pte)
  2060. {
  2061. swp_entry_t swp;
  2062. if (huge_pte_none(pte) || pte_present(pte))
  2063. return 0;
  2064. swp = pte_to_swp_entry(pte);
  2065. if (non_swap_entry(swp) && is_hwpoison_entry(swp))
  2066. return 1;
  2067. else
  2068. return 0;
  2069. }
  2070. void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
  2071. unsigned long start, unsigned long end,
  2072. struct page *ref_page)
  2073. {
  2074. int force_flush = 0;
  2075. struct mm_struct *mm = vma->vm_mm;
  2076. unsigned long address;
  2077. pte_t *ptep;
  2078. pte_t pte;
  2079. spinlock_t *ptl;
  2080. struct page *page;
  2081. struct hstate *h = hstate_vma(vma);
  2082. unsigned long sz = huge_page_size(h);
  2083. const unsigned long mmun_start = start; /* For mmu_notifiers */
  2084. const unsigned long mmun_end = end; /* For mmu_notifiers */
  2085. WARN_ON(!is_vm_hugetlb_page(vma));
  2086. BUG_ON(start & ~huge_page_mask(h));
  2087. BUG_ON(end & ~huge_page_mask(h));
  2088. tlb_start_vma(tlb, vma);
  2089. mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
  2090. again:
  2091. for (address = start; address < end; address += sz) {
  2092. ptep = huge_pte_offset(mm, address);
  2093. if (!ptep)
  2094. continue;
  2095. ptl = huge_pte_lock(h, mm, ptep);
  2096. if (huge_pmd_unshare(mm, &address, ptep))
  2097. goto unlock;
  2098. pte = huge_ptep_get(ptep);
  2099. if (huge_pte_none(pte))
  2100. goto unlock;
  2101. /*
  2102. * HWPoisoned hugepage is already unmapped and dropped reference
  2103. */
  2104. if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
  2105. huge_pte_clear(mm, address, ptep);
  2106. goto unlock;
  2107. }
  2108. page = pte_page(pte);
  2109. /*
  2110. * If a reference page is supplied, it is because a specific
  2111. * page is being unmapped, not a range. Ensure the page we
  2112. * are about to unmap is the actual page of interest.
  2113. */
  2114. if (ref_page) {
  2115. if (page != ref_page)
  2116. goto unlock;
  2117. /*
  2118. * Mark the VMA as having unmapped its page so that
  2119. * future faults in this VMA will fail rather than
  2120. * looking like data was lost
  2121. */
  2122. set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
  2123. }
  2124. pte = huge_ptep_get_and_clear(mm, address, ptep);
  2125. tlb_remove_tlb_entry(tlb, ptep, address);
  2126. if (huge_pte_dirty(pte))
  2127. set_page_dirty(page);
  2128. page_remove_rmap(page);
  2129. force_flush = !__tlb_remove_page(tlb, page);
  2130. if (force_flush) {
  2131. spin_unlock(ptl);
  2132. break;
  2133. }
  2134. /* Bail out after unmapping reference page if supplied */
  2135. if (ref_page) {
  2136. spin_unlock(ptl);
  2137. break;
  2138. }
  2139. unlock:
  2140. spin_unlock(ptl);
  2141. }
  2142. /*
  2143. * mmu_gather ran out of room to batch pages, we break out of
  2144. * the PTE lock to avoid doing the potential expensive TLB invalidate
  2145. * and page-free while holding it.
  2146. */
  2147. if (force_flush) {
  2148. force_flush = 0;
  2149. tlb_flush_mmu(tlb);
  2150. if (address < end && !ref_page)
  2151. goto again;
  2152. }
  2153. mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
  2154. tlb_end_vma(tlb, vma);
  2155. }
  2156. void __unmap_hugepage_range_final(struct mmu_gather *tlb,
  2157. struct vm_area_struct *vma, unsigned long start,
  2158. unsigned long end, struct page *ref_page)
  2159. {
  2160. __unmap_hugepage_range(tlb, vma, start, end, ref_page);
  2161. /*
  2162. * Clear this flag so that x86's huge_pmd_share page_table_shareable
  2163. * test will fail on a vma being torn down, and not grab a page table
  2164. * on its way out. We're lucky that the flag has such an appropriate
  2165. * name, and can in fact be safely cleared here. We could clear it
  2166. * before the __unmap_hugepage_range above, but all that's necessary
  2167. * is to clear it before releasing the i_mmap_mutex. This works
  2168. * because in the context this is called, the VMA is about to be
  2169. * destroyed and the i_mmap_mutex is held.
  2170. */
  2171. vma->vm_flags &= ~VM_MAYSHARE;
  2172. }
  2173. void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
  2174. unsigned long end, struct page *ref_page)
  2175. {
  2176. struct mm_struct *mm;
  2177. struct mmu_gather tlb;
  2178. mm = vma->vm_mm;
  2179. tlb_gather_mmu(&tlb, mm, start, end);
  2180. __unmap_hugepage_range(&tlb, vma, start, end, ref_page);
  2181. tlb_finish_mmu(&tlb, start, end);
  2182. }
  2183. /*
  2184. * This is called when the original mapper is failing to COW a MAP_PRIVATE
  2185. * mappping it owns the reserve page for. The intention is to unmap the page
  2186. * from other VMAs and let the children be SIGKILLed if they are faulting the
  2187. * same region.
  2188. */
  2189. static int unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
  2190. struct page *page, unsigned long address)
  2191. {
  2192. struct hstate *h = hstate_vma(vma);
  2193. struct vm_area_struct *iter_vma;
  2194. struct address_space *mapping;
  2195. pgoff_t pgoff;
  2196. /*
  2197. * vm_pgoff is in PAGE_SIZE units, hence the different calculation
  2198. * from page cache lookup which is in HPAGE_SIZE units.
  2199. */
  2200. address = address & huge_page_mask(h);
  2201. pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
  2202. vma->vm_pgoff;
  2203. mapping = file_inode(vma->vm_file)->i_mapping;
  2204. /*
  2205. * Take the mapping lock for the duration of the table walk. As
  2206. * this mapping should be shared between all the VMAs,
  2207. * __unmap_hugepage_range() is called as the lock is already held
  2208. */
  2209. mutex_lock(&mapping->i_mmap_mutex);
  2210. vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
  2211. /* Do not unmap the current VMA */
  2212. if (iter_vma == vma)
  2213. continue;
  2214. /*
  2215. * Unmap the page from other VMAs without their own reserves.
  2216. * They get marked to be SIGKILLed if they fault in these
  2217. * areas. This is because a future no-page fault on this VMA
  2218. * could insert a zeroed page instead of the data existing
  2219. * from the time of fork. This would look like data corruption
  2220. */
  2221. if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
  2222. unmap_hugepage_range(iter_vma, address,
  2223. address + huge_page_size(h), page);
  2224. }
  2225. mutex_unlock(&mapping->i_mmap_mutex);
  2226. return 1;
  2227. }
  2228. /*
  2229. * Hugetlb_cow() should be called with page lock of the original hugepage held.
  2230. * Called with hugetlb_instantiation_mutex held and pte_page locked so we
  2231. * cannot race with other handlers or page migration.
  2232. * Keep the pte_same checks anyway to make transition from the mutex easier.
  2233. */
  2234. static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
  2235. unsigned long address, pte_t *ptep, pte_t pte,
  2236. struct page *pagecache_page, spinlock_t *ptl)
  2237. {
  2238. struct hstate *h = hstate_vma(vma);
  2239. struct page *old_page, *new_page;
  2240. int outside_reserve = 0;
  2241. unsigned long mmun_start; /* For mmu_notifiers */
  2242. unsigned long mmun_end; /* For mmu_notifiers */
  2243. old_page = pte_page(pte);
  2244. retry_avoidcopy:
  2245. /* If no-one else is actually using this page, avoid the copy
  2246. * and just make the page writable */
  2247. if (page_mapcount(old_page) == 1 && PageAnon(old_page)) {
  2248. page_move_anon_rmap(old_page, vma, address);
  2249. set_huge_ptep_writable(vma, address, ptep);
  2250. return 0;
  2251. }
  2252. /*
  2253. * If the process that created a MAP_PRIVATE mapping is about to
  2254. * perform a COW due to a shared page count, attempt to satisfy
  2255. * the allocation without using the existing reserves. The pagecache
  2256. * page is used to determine if the reserve at this address was
  2257. * consumed or not. If reserves were used, a partial faulted mapping
  2258. * at the time of fork() could consume its reserves on COW instead
  2259. * of the full address range.
  2260. */
  2261. if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
  2262. old_page != pagecache_page)
  2263. outside_reserve = 1;
  2264. page_cache_get(old_page);
  2265. /* Drop page table lock as buddy allocator may be called */
  2266. spin_unlock(ptl);
  2267. new_page = alloc_huge_page(vma, address, outside_reserve);
  2268. if (IS_ERR(new_page)) {
  2269. long err = PTR_ERR(new_page);
  2270. page_cache_release(old_page);
  2271. /*
  2272. * If a process owning a MAP_PRIVATE mapping fails to COW,
  2273. * it is due to references held by a child and an insufficient
  2274. * huge page pool. To guarantee the original mappers
  2275. * reliability, unmap the page from child processes. The child
  2276. * may get SIGKILLed if it later faults.
  2277. */
  2278. if (outside_reserve) {
  2279. BUG_ON(huge_pte_none(pte));
  2280. if (unmap_ref_private(mm, vma, old_page, address)) {
  2281. BUG_ON(huge_pte_none(pte));
  2282. spin_lock(ptl);
  2283. ptep = huge_pte_offset(mm, address & huge_page_mask(h));
  2284. if (likely(pte_same(huge_ptep_get(ptep), pte)))
  2285. goto retry_avoidcopy;
  2286. /*
  2287. * race occurs while re-acquiring page table
  2288. * lock, and our job is done.
  2289. */
  2290. return 0;
  2291. }
  2292. WARN_ON_ONCE(1);
  2293. }
  2294. /* Caller expects lock to be held */
  2295. spin_lock(ptl);
  2296. if (err == -ENOMEM)
  2297. return VM_FAULT_OOM;
  2298. else
  2299. return VM_FAULT_SIGBUS;
  2300. }
  2301. /*
  2302. * When the original hugepage is shared one, it does not have
  2303. * anon_vma prepared.
  2304. */
  2305. if (unlikely(anon_vma_prepare(vma))) {
  2306. page_cache_release(new_page);
  2307. page_cache_release(old_page);
  2308. /* Caller expects lock to be held */
  2309. spin_lock(ptl);
  2310. return VM_FAULT_OOM;
  2311. }
  2312. copy_user_huge_page(new_page, old_page, address, vma,
  2313. pages_per_huge_page(h));
  2314. __SetPageUptodate(new_page);
  2315. mmun_start = address & huge_page_mask(h);
  2316. mmun_end = mmun_start + huge_page_size(h);
  2317. mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
  2318. /*
  2319. * Retake the page table lock to check for racing updates
  2320. * before the page tables are altered
  2321. */
  2322. spin_lock(ptl);
  2323. ptep = huge_pte_offset(mm, address & huge_page_mask(h));
  2324. if (likely(pte_same(huge_ptep_get(ptep), pte))) {
  2325. ClearPagePrivate(new_page);
  2326. /* Break COW */
  2327. huge_ptep_clear_flush(vma, address, ptep);
  2328. set_huge_pte_at(mm, address, ptep,
  2329. make_huge_pte(vma, new_page, 1));
  2330. page_remove_rmap(old_page);
  2331. hugepage_add_new_anon_rmap(new_page, vma, address);
  2332. /* Make the old page be freed below */
  2333. new_page = old_page;
  2334. }
  2335. spin_unlock(ptl);
  2336. mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
  2337. page_cache_release(new_page);
  2338. page_cache_release(old_page);
  2339. /* Caller expects lock to be held */
  2340. spin_lock(ptl);
  2341. return 0;
  2342. }
  2343. /* Return the pagecache page at a given address within a VMA */
  2344. static struct page *hugetlbfs_pagecache_page(struct hstate *h,
  2345. struct vm_area_struct *vma, unsigned long address)
  2346. {
  2347. struct address_space *mapping;
  2348. pgoff_t idx;
  2349. mapping = vma->vm_file->f_mapping;
  2350. idx = vma_hugecache_offset(h, vma, address);
  2351. return find_lock_page(mapping, idx);
  2352. }
  2353. /*
  2354. * Return whether there is a pagecache page to back given address within VMA.
  2355. * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
  2356. */
  2357. static bool hugetlbfs_pagecache_present(struct hstate *h,
  2358. struct vm_area_struct *vma, unsigned long address)
  2359. {
  2360. struct address_space *mapping;
  2361. pgoff_t idx;
  2362. struct page *page;
  2363. mapping = vma->vm_file->f_mapping;
  2364. idx = vma_hugecache_offset(h, vma, address);
  2365. page = find_get_page(mapping, idx);
  2366. if (page)
  2367. put_page(page);
  2368. return page != NULL;
  2369. }
  2370. static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
  2371. unsigned long address, pte_t *ptep, unsigned int flags)
  2372. {
  2373. struct hstate *h = hstate_vma(vma);
  2374. int ret = VM_FAULT_SIGBUS;
  2375. int anon_rmap = 0;
  2376. pgoff_t idx;
  2377. unsigned long size;
  2378. struct page *page;
  2379. struct address_space *mapping;
  2380. pte_t new_pte;
  2381. spinlock_t *ptl;
  2382. /*
  2383. * Currently, we are forced to kill the process in the event the
  2384. * original mapper has unmapped pages from the child due to a failed
  2385. * COW. Warn that such a situation has occurred as it may not be obvious
  2386. */
  2387. if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
  2388. pr_warning("PID %d killed due to inadequate hugepage pool\n",
  2389. current->pid);
  2390. return ret;
  2391. }
  2392. mapping = vma->vm_file->f_mapping;
  2393. idx = vma_hugecache_offset(h, vma, address);
  2394. /*
  2395. * Use page lock to guard against racing truncation
  2396. * before we get page_table_lock.
  2397. */
  2398. retry:
  2399. page = find_lock_page(mapping, idx);
  2400. if (!page) {
  2401. size = i_size_read(mapping->host) >> huge_page_shift(h);
  2402. if (idx >= size)
  2403. goto out;
  2404. page = alloc_huge_page(vma, address, 0);
  2405. if (IS_ERR(page)) {
  2406. ret = PTR_ERR(page);
  2407. if (ret == -ENOMEM)
  2408. ret = VM_FAULT_OOM;
  2409. else
  2410. ret = VM_FAULT_SIGBUS;
  2411. goto out;
  2412. }
  2413. clear_huge_page(page, address, pages_per_huge_page(h));
  2414. __SetPageUptodate(page);
  2415. if (vma->vm_flags & VM_MAYSHARE) {
  2416. int err;
  2417. struct inode *inode = mapping->host;
  2418. err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
  2419. if (err) {
  2420. put_page(page);
  2421. if (err == -EEXIST)
  2422. goto retry;
  2423. goto out;
  2424. }
  2425. ClearPagePrivate(page);
  2426. spin_lock(&inode->i_lock);
  2427. inode->i_blocks += blocks_per_huge_page(h);
  2428. spin_unlock(&inode->i_lock);
  2429. } else {
  2430. lock_page(page);
  2431. if (unlikely(anon_vma_prepare(vma))) {
  2432. ret = VM_FAULT_OOM;
  2433. goto backout_unlocked;
  2434. }
  2435. anon_rmap = 1;
  2436. }
  2437. } else {
  2438. /*
  2439. * If memory error occurs between mmap() and fault, some process
  2440. * don't have hwpoisoned swap entry for errored virtual address.
  2441. * So we need to block hugepage fault by PG_hwpoison bit check.
  2442. */
  2443. if (unlikely(PageHWPoison(page))) {
  2444. ret = VM_FAULT_HWPOISON |
  2445. VM_FAULT_SET_HINDEX(hstate_index(h));
  2446. goto backout_unlocked;
  2447. }
  2448. }
  2449. /*
  2450. * If we are going to COW a private mapping later, we examine the
  2451. * pending reservations for this page now. This will ensure that
  2452. * any allocations necessary to record that reservation occur outside
  2453. * the spinlock.
  2454. */
  2455. if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED))
  2456. if (vma_needs_reservation(h, vma, address) < 0) {
  2457. ret = VM_FAULT_OOM;
  2458. goto backout_unlocked;
  2459. }
  2460. ptl = huge_pte_lockptr(h, mm, ptep);
  2461. spin_lock(ptl);
  2462. size = i_size_read(mapping->host) >> huge_page_shift(h);
  2463. if (idx >= size)
  2464. goto backout;
  2465. ret = 0;
  2466. if (!huge_pte_none(huge_ptep_get(ptep)))
  2467. goto backout;
  2468. if (anon_rmap) {
  2469. ClearPagePrivate(page);
  2470. hugepage_add_new_anon_rmap(page, vma, address);
  2471. }
  2472. else
  2473. page_dup_rmap(page);
  2474. new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
  2475. && (vma->vm_flags & VM_SHARED)));
  2476. set_huge_pte_at(mm, address, ptep, new_pte);
  2477. if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
  2478. /* Optimization, do the COW without a second fault */
  2479. ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page, ptl);
  2480. }
  2481. spin_unlock(ptl);
  2482. unlock_page(page);
  2483. out:
  2484. return ret;
  2485. backout:
  2486. spin_unlock(ptl);
  2487. backout_unlocked:
  2488. unlock_page(page);
  2489. put_page(page);
  2490. goto out;
  2491. }
  2492. int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
  2493. unsigned long address, unsigned int flags)
  2494. {
  2495. pte_t *ptep;
  2496. pte_t entry;
  2497. spinlock_t *ptl;
  2498. int ret;
  2499. struct page *page = NULL;
  2500. struct page *pagecache_page = NULL;
  2501. static DEFINE_MUTEX(hugetlb_instantiation_mutex);
  2502. struct hstate *h = hstate_vma(vma);
  2503. address &= huge_page_mask(h);
  2504. ptep = huge_pte_offset(mm, address);
  2505. if (ptep) {
  2506. entry = huge_ptep_get(ptep);
  2507. if (unlikely(is_hugetlb_entry_migration(entry))) {
  2508. migration_entry_wait_huge(vma, mm, ptep);
  2509. return 0;
  2510. } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
  2511. return VM_FAULT_HWPOISON_LARGE |
  2512. VM_FAULT_SET_HINDEX(hstate_index(h));
  2513. }
  2514. ptep = huge_pte_alloc(mm, address, huge_page_size(h));
  2515. if (!ptep)
  2516. return VM_FAULT_OOM;
  2517. /*
  2518. * Serialize hugepage allocation and instantiation, so that we don't
  2519. * get spurious allocation failures if two CPUs race to instantiate
  2520. * the same page in the page cache.
  2521. */
  2522. mutex_lock(&hugetlb_instantiation_mutex);
  2523. entry = huge_ptep_get(ptep);
  2524. if (huge_pte_none(entry)) {
  2525. ret = hugetlb_no_page(mm, vma, address, ptep, flags);
  2526. goto out_mutex;
  2527. }
  2528. ret = 0;
  2529. /*
  2530. * If we are going to COW the mapping later, we examine the pending
  2531. * reservations for this page now. This will ensure that any
  2532. * allocations necessary to record that reservation occur outside the
  2533. * spinlock. For private mappings, we also lookup the pagecache
  2534. * page now as it is used to determine if a reservation has been
  2535. * consumed.
  2536. */
  2537. if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
  2538. if (vma_needs_reservation(h, vma, address) < 0) {
  2539. ret = VM_FAULT_OOM;
  2540. goto out_mutex;
  2541. }
  2542. if (!(vma->vm_flags & VM_MAYSHARE))
  2543. pagecache_page = hugetlbfs_pagecache_page(h,
  2544. vma, address);
  2545. }
  2546. /*
  2547. * hugetlb_cow() requires page locks of pte_page(entry) and
  2548. * pagecache_page, so here we need take the former one
  2549. * when page != pagecache_page or !pagecache_page.
  2550. * Note that locking order is always pagecache_page -> page,
  2551. * so no worry about deadlock.
  2552. */
  2553. page = pte_page(entry);
  2554. get_page(page);
  2555. if (page != pagecache_page)
  2556. lock_page(page);
  2557. ptl = huge_pte_lockptr(h, mm, ptep);
  2558. spin_lock(ptl);
  2559. /* Check for a racing update before calling hugetlb_cow */
  2560. if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
  2561. goto out_ptl;
  2562. if (flags & FAULT_FLAG_WRITE) {
  2563. if (!huge_pte_write(entry)) {
  2564. ret = hugetlb_cow(mm, vma, address, ptep, entry,
  2565. pagecache_page, ptl);
  2566. goto out_ptl;
  2567. }
  2568. entry = huge_pte_mkdirty(entry);
  2569. }
  2570. entry = pte_mkyoung(entry);
  2571. if (huge_ptep_set_access_flags(vma, address, ptep, entry,
  2572. flags & FAULT_FLAG_WRITE))
  2573. update_mmu_cache(vma, address, ptep);
  2574. out_ptl:
  2575. spin_unlock(ptl);
  2576. if (pagecache_page) {
  2577. unlock_page(pagecache_page);
  2578. put_page(pagecache_page);
  2579. }
  2580. if (page != pagecache_page)
  2581. unlock_page(page);
  2582. put_page(page);
  2583. out_mutex:
  2584. mutex_unlock(&hugetlb_instantiation_mutex);
  2585. return ret;
  2586. }
  2587. long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
  2588. struct page **pages, struct vm_area_struct **vmas,
  2589. unsigned long *position, unsigned long *nr_pages,
  2590. long i, unsigned int flags)
  2591. {
  2592. unsigned long pfn_offset;
  2593. unsigned long vaddr = *position;
  2594. unsigned long remainder = *nr_pages;
  2595. struct hstate *h = hstate_vma(vma);
  2596. while (vaddr < vma->vm_end && remainder) {
  2597. pte_t *pte;
  2598. spinlock_t *ptl = NULL;
  2599. int absent;
  2600. struct page *page;
  2601. /*
  2602. * Some archs (sparc64, sh*) have multiple pte_ts to
  2603. * each hugepage. We have to make sure we get the
  2604. * first, for the page indexing below to work.
  2605. *
  2606. * Note that page table lock is not held when pte is null.
  2607. */
  2608. pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
  2609. if (pte)
  2610. ptl = huge_pte_lock(h, mm, pte);
  2611. absent = !pte || huge_pte_none(huge_ptep_get(pte));
  2612. /*
  2613. * When coredumping, it suits get_dump_page if we just return
  2614. * an error where there's an empty slot with no huge pagecache
  2615. * to back it. This way, we avoid allocating a hugepage, and
  2616. * the sparse dumpfile avoids allocating disk blocks, but its
  2617. * huge holes still show up with zeroes where they need to be.
  2618. */
  2619. if (absent && (flags & FOLL_DUMP) &&
  2620. !hugetlbfs_pagecache_present(h, vma, vaddr)) {
  2621. if (pte)
  2622. spin_unlock(ptl);
  2623. remainder = 0;
  2624. break;
  2625. }
  2626. /*
  2627. * We need call hugetlb_fault for both hugepages under migration
  2628. * (in which case hugetlb_fault waits for the migration,) and
  2629. * hwpoisoned hugepages (in which case we need to prevent the
  2630. * caller from accessing to them.) In order to do this, we use
  2631. * here is_swap_pte instead of is_hugetlb_entry_migration and
  2632. * is_hugetlb_entry_hwpoisoned. This is because it simply covers
  2633. * both cases, and because we can't follow correct pages
  2634. * directly from any kind of swap entries.
  2635. */
  2636. if (absent || is_swap_pte(huge_ptep_get(pte)) ||
  2637. ((flags & FOLL_WRITE) &&
  2638. !huge_pte_write(huge_ptep_get(pte)))) {
  2639. int ret;
  2640. if (pte)
  2641. spin_unlock(ptl);
  2642. ret = hugetlb_fault(mm, vma, vaddr,
  2643. (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
  2644. if (!(ret & VM_FAULT_ERROR))
  2645. continue;
  2646. remainder = 0;
  2647. break;
  2648. }
  2649. pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
  2650. page = pte_page(huge_ptep_get(pte));
  2651. same_page:
  2652. if (pages) {
  2653. pages[i] = mem_map_offset(page, pfn_offset);
  2654. get_page(pages[i]);
  2655. }
  2656. if (vmas)
  2657. vmas[i] = vma;
  2658. vaddr += PAGE_SIZE;
  2659. ++pfn_offset;
  2660. --remainder;
  2661. ++i;
  2662. if (vaddr < vma->vm_end && remainder &&
  2663. pfn_offset < pages_per_huge_page(h)) {
  2664. /*
  2665. * We use pfn_offset to avoid touching the pageframes
  2666. * of this compound page.
  2667. */
  2668. goto same_page;
  2669. }
  2670. spin_unlock(ptl);
  2671. }
  2672. *nr_pages = remainder;
  2673. *position = vaddr;
  2674. return i ? i : -EFAULT;
  2675. }
  2676. unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
  2677. unsigned long address, unsigned long end, pgprot_t newprot)
  2678. {
  2679. struct mm_struct *mm = vma->vm_mm;
  2680. unsigned long start = address;
  2681. pte_t *ptep;
  2682. pte_t pte;
  2683. struct hstate *h = hstate_vma(vma);
  2684. unsigned long pages = 0;
  2685. BUG_ON(address >= end);
  2686. flush_cache_range(vma, address, end);
  2687. mutex_lock(&vma->vm_file->f_mapping->i_mmap_mutex);
  2688. for (; address < end; address += huge_page_size(h)) {
  2689. spinlock_t *ptl;
  2690. ptep = huge_pte_offset(mm, address);
  2691. if (!ptep)
  2692. continue;
  2693. ptl = huge_pte_lock(h, mm, ptep);
  2694. if (huge_pmd_unshare(mm, &address, ptep)) {
  2695. pages++;
  2696. spin_unlock(ptl);
  2697. continue;
  2698. }
  2699. if (!huge_pte_none(huge_ptep_get(ptep))) {
  2700. pte = huge_ptep_get_and_clear(mm, address, ptep);
  2701. pte = pte_mkhuge(huge_pte_modify(pte, newprot));
  2702. pte = arch_make_huge_pte(pte, vma, NULL, 0);
  2703. set_huge_pte_at(mm, address, ptep, pte);
  2704. pages++;
  2705. }
  2706. spin_unlock(ptl);
  2707. }
  2708. /*
  2709. * Must flush TLB before releasing i_mmap_mutex: x86's huge_pmd_unshare
  2710. * may have cleared our pud entry and done put_page on the page table:
  2711. * once we release i_mmap_mutex, another task can do the final put_page
  2712. * and that page table be reused and filled with junk.
  2713. */
  2714. flush_tlb_range(vma, start, end);
  2715. mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex);
  2716. return pages << h->order;
  2717. }
  2718. int hugetlb_reserve_pages(struct inode *inode,
  2719. long from, long to,
  2720. struct vm_area_struct *vma,
  2721. vm_flags_t vm_flags)
  2722. {
  2723. long ret, chg;
  2724. struct hstate *h = hstate_inode(inode);
  2725. struct hugepage_subpool *spool = subpool_inode(inode);
  2726. /*
  2727. * Only apply hugepage reservation if asked. At fault time, an
  2728. * attempt will be made for VM_NORESERVE to allocate a page
  2729. * without using reserves
  2730. */
  2731. if (vm_flags & VM_NORESERVE)
  2732. return 0;
  2733. /*
  2734. * Shared mappings base their reservation on the number of pages that
  2735. * are already allocated on behalf of the file. Private mappings need
  2736. * to reserve the full area even if read-only as mprotect() may be
  2737. * called to make the mapping read-write. Assume !vma is a shm mapping
  2738. */
  2739. if (!vma || vma->vm_flags & VM_MAYSHARE)
  2740. chg = region_chg(&inode->i_mapping->private_list, from, to);
  2741. else {
  2742. struct resv_map *resv_map = resv_map_alloc();
  2743. if (!resv_map)
  2744. return -ENOMEM;
  2745. chg = to - from;
  2746. set_vma_resv_map(vma, resv_map);
  2747. set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
  2748. }
  2749. if (chg < 0) {
  2750. ret = chg;
  2751. goto out_err;
  2752. }
  2753. /* There must be enough pages in the subpool for the mapping */
  2754. if (hugepage_subpool_get_pages(spool, chg)) {
  2755. ret = -ENOSPC;
  2756. goto out_err;
  2757. }
  2758. /*
  2759. * Check enough hugepages are available for the reservation.
  2760. * Hand the pages back to the subpool if there are not
  2761. */
  2762. ret = hugetlb_acct_memory(h, chg);
  2763. if (ret < 0) {
  2764. hugepage_subpool_put_pages(spool, chg);
  2765. goto out_err;
  2766. }
  2767. /*
  2768. * Account for the reservations made. Shared mappings record regions
  2769. * that have reservations as they are shared by multiple VMAs.
  2770. * When the last VMA disappears, the region map says how much
  2771. * the reservation was and the page cache tells how much of
  2772. * the reservation was consumed. Private mappings are per-VMA and
  2773. * only the consumed reservations are tracked. When the VMA
  2774. * disappears, the original reservation is the VMA size and the
  2775. * consumed reservations are stored in the map. Hence, nothing
  2776. * else has to be done for private mappings here
  2777. */
  2778. if (!vma || vma->vm_flags & VM_MAYSHARE)
  2779. region_add(&inode->i_mapping->private_list, from, to);
  2780. return 0;
  2781. out_err:
  2782. if (vma)
  2783. resv_map_put(vma);
  2784. return ret;
  2785. }
  2786. void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
  2787. {
  2788. struct hstate *h = hstate_inode(inode);
  2789. long chg = region_truncate(&inode->i_mapping->private_list, offset);
  2790. struct hugepage_subpool *spool = subpool_inode(inode);
  2791. spin_lock(&inode->i_lock);
  2792. inode->i_blocks -= (blocks_per_huge_page(h) * freed);
  2793. spin_unlock(&inode->i_lock);
  2794. hugepage_subpool_put_pages(spool, (chg - freed));
  2795. hugetlb_acct_memory(h, -(chg - freed));
  2796. }
  2797. #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
  2798. static unsigned long page_table_shareable(struct vm_area_struct *svma,
  2799. struct vm_area_struct *vma,
  2800. unsigned long addr, pgoff_t idx)
  2801. {
  2802. unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
  2803. svma->vm_start;
  2804. unsigned long sbase = saddr & PUD_MASK;
  2805. unsigned long s_end = sbase + PUD_SIZE;
  2806. /* Allow segments to share if only one is marked locked */
  2807. unsigned long vm_flags = vma->vm_flags & ~VM_LOCKED;
  2808. unsigned long svm_flags = svma->vm_flags & ~VM_LOCKED;
  2809. /*
  2810. * match the virtual addresses, permission and the alignment of the
  2811. * page table page.
  2812. */
  2813. if (pmd_index(addr) != pmd_index(saddr) ||
  2814. vm_flags != svm_flags ||
  2815. sbase < svma->vm_start || svma->vm_end < s_end)
  2816. return 0;
  2817. return saddr;
  2818. }
  2819. static int vma_shareable(struct vm_area_struct *vma, unsigned long addr)
  2820. {
  2821. unsigned long base = addr & PUD_MASK;
  2822. unsigned long end = base + PUD_SIZE;
  2823. /*
  2824. * check on proper vm_flags and page table alignment
  2825. */
  2826. if (vma->vm_flags & VM_MAYSHARE &&
  2827. vma->vm_start <= base && end <= vma->vm_end)
  2828. return 1;
  2829. return 0;
  2830. }
  2831. /*
  2832. * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
  2833. * and returns the corresponding pte. While this is not necessary for the
  2834. * !shared pmd case because we can allocate the pmd later as well, it makes the
  2835. * code much cleaner. pmd allocation is essential for the shared case because
  2836. * pud has to be populated inside the same i_mmap_mutex section - otherwise
  2837. * racing tasks could either miss the sharing (see huge_pte_offset) or select a
  2838. * bad pmd for sharing.
  2839. */
  2840. pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
  2841. {
  2842. struct vm_area_struct *vma = find_vma(mm, addr);
  2843. struct address_space *mapping = vma->vm_file->f_mapping;
  2844. pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
  2845. vma->vm_pgoff;
  2846. struct vm_area_struct *svma;
  2847. unsigned long saddr;
  2848. pte_t *spte = NULL;
  2849. pte_t *pte;
  2850. spinlock_t *ptl;
  2851. if (!vma_shareable(vma, addr))
  2852. return (pte_t *)pmd_alloc(mm, pud, addr);
  2853. mutex_lock(&mapping->i_mmap_mutex);
  2854. vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
  2855. if (svma == vma)
  2856. continue;
  2857. saddr = page_table_shareable(svma, vma, addr, idx);
  2858. if (saddr) {
  2859. spte = huge_pte_offset(svma->vm_mm, saddr);
  2860. if (spte) {
  2861. get_page(virt_to_page(spte));
  2862. break;
  2863. }
  2864. }
  2865. }
  2866. if (!spte)
  2867. goto out;
  2868. ptl = huge_pte_lockptr(hstate_vma(vma), mm, spte);
  2869. spin_lock(ptl);
  2870. if (pud_none(*pud))
  2871. pud_populate(mm, pud,
  2872. (pmd_t *)((unsigned long)spte & PAGE_MASK));
  2873. else
  2874. put_page(virt_to_page(spte));
  2875. spin_unlock(ptl);
  2876. out:
  2877. pte = (pte_t *)pmd_alloc(mm, pud, addr);
  2878. mutex_unlock(&mapping->i_mmap_mutex);
  2879. return pte;
  2880. }
  2881. /*
  2882. * unmap huge page backed by shared pte.
  2883. *
  2884. * Hugetlb pte page is ref counted at the time of mapping. If pte is shared
  2885. * indicated by page_count > 1, unmap is achieved by clearing pud and
  2886. * decrementing the ref count. If count == 1, the pte page is not shared.
  2887. *
  2888. * called with page table lock held.
  2889. *
  2890. * returns: 1 successfully unmapped a shared pte page
  2891. * 0 the underlying pte page is not shared, or it is the last user
  2892. */
  2893. int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
  2894. {
  2895. pgd_t *pgd = pgd_offset(mm, *addr);
  2896. pud_t *pud = pud_offset(pgd, *addr);
  2897. BUG_ON(page_count(virt_to_page(ptep)) == 0);
  2898. if (page_count(virt_to_page(ptep)) == 1)
  2899. return 0;
  2900. pud_clear(pud);
  2901. put_page(virt_to_page(ptep));
  2902. *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE;
  2903. return 1;
  2904. }
  2905. #define want_pmd_share() (1)
  2906. #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
  2907. pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
  2908. {
  2909. return NULL;
  2910. }
  2911. #define want_pmd_share() (0)
  2912. #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
  2913. #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
  2914. pte_t *huge_pte_alloc(struct mm_struct *mm,
  2915. unsigned long addr, unsigned long sz)
  2916. {
  2917. pgd_t *pgd;
  2918. pud_t *pud;
  2919. pte_t *pte = NULL;
  2920. pgd = pgd_offset(mm, addr);
  2921. pud = pud_alloc(mm, pgd, addr);
  2922. if (pud) {
  2923. if (sz == PUD_SIZE) {
  2924. pte = (pte_t *)pud;
  2925. } else {
  2926. BUG_ON(sz != PMD_SIZE);
  2927. if (want_pmd_share() && pud_none(*pud))
  2928. pte = huge_pmd_share(mm, addr, pud);
  2929. else
  2930. pte = (pte_t *)pmd_alloc(mm, pud, addr);
  2931. }
  2932. }
  2933. BUG_ON(pte && !pte_none(*pte) && !pte_huge(*pte));
  2934. return pte;
  2935. }
  2936. pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
  2937. {
  2938. pgd_t *pgd;
  2939. pud_t *pud;
  2940. pmd_t *pmd = NULL;
  2941. pgd = pgd_offset(mm, addr);
  2942. if (pgd_present(*pgd)) {
  2943. pud = pud_offset(pgd, addr);
  2944. if (pud_present(*pud)) {
  2945. if (pud_huge(*pud))
  2946. return (pte_t *)pud;
  2947. pmd = pmd_offset(pud, addr);
  2948. }
  2949. }
  2950. return (pte_t *) pmd;
  2951. }
  2952. struct page *
  2953. follow_huge_pmd(struct mm_struct *mm, unsigned long address,
  2954. pmd_t *pmd, int write)
  2955. {
  2956. struct page *page;
  2957. page = pte_page(*(pte_t *)pmd);
  2958. if (page)
  2959. page += ((address & ~PMD_MASK) >> PAGE_SHIFT);
  2960. return page;
  2961. }
  2962. struct page *
  2963. follow_huge_pud(struct mm_struct *mm, unsigned long address,
  2964. pud_t *pud, int write)
  2965. {
  2966. struct page *page;
  2967. page = pte_page(*(pte_t *)pud);
  2968. if (page)
  2969. page += ((address & ~PUD_MASK) >> PAGE_SHIFT);
  2970. return page;
  2971. }
  2972. #else /* !CONFIG_ARCH_WANT_GENERAL_HUGETLB */
  2973. /* Can be overriden by architectures */
  2974. __attribute__((weak)) struct page *
  2975. follow_huge_pud(struct mm_struct *mm, unsigned long address,
  2976. pud_t *pud, int write)
  2977. {
  2978. BUG();
  2979. return NULL;
  2980. }
  2981. #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
  2982. #ifdef CONFIG_MEMORY_FAILURE
  2983. /* Should be called in hugetlb_lock */
  2984. static int is_hugepage_on_freelist(struct page *hpage)
  2985. {
  2986. struct page *page;
  2987. struct page *tmp;
  2988. struct hstate *h = page_hstate(hpage);
  2989. int nid = page_to_nid(hpage);
  2990. list_for_each_entry_safe(page, tmp, &h->hugepage_freelists[nid], lru)
  2991. if (page == hpage)
  2992. return 1;
  2993. return 0;
  2994. }
  2995. /*
  2996. * This function is called from memory failure code.
  2997. * Assume the caller holds page lock of the head page.
  2998. */
  2999. int dequeue_hwpoisoned_huge_page(struct page *hpage)
  3000. {
  3001. struct hstate *h = page_hstate(hpage);
  3002. int nid = page_to_nid(hpage);
  3003. int ret = -EBUSY;
  3004. spin_lock(&hugetlb_lock);
  3005. if (is_hugepage_on_freelist(hpage)) {
  3006. /*
  3007. * Hwpoisoned hugepage isn't linked to activelist or freelist,
  3008. * but dangling hpage->lru can trigger list-debug warnings
  3009. * (this happens when we call unpoison_memory() on it),
  3010. * so let it point to itself with list_del_init().
  3011. */
  3012. list_del_init(&hpage->lru);
  3013. set_page_refcounted(hpage);
  3014. h->free_huge_pages--;
  3015. h->free_huge_pages_node[nid]--;
  3016. ret = 0;
  3017. }
  3018. spin_unlock(&hugetlb_lock);
  3019. return ret;
  3020. }
  3021. #endif
  3022. bool isolate_huge_page(struct page *page, struct list_head *list)
  3023. {
  3024. VM_BUG_ON(!PageHead(page));
  3025. if (!get_page_unless_zero(page))
  3026. return false;
  3027. spin_lock(&hugetlb_lock);
  3028. list_move_tail(&page->lru, list);
  3029. spin_unlock(&hugetlb_lock);
  3030. return true;
  3031. }
  3032. void putback_active_hugepage(struct page *page)
  3033. {
  3034. VM_BUG_ON(!PageHead(page));
  3035. spin_lock(&hugetlb_lock);
  3036. list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
  3037. spin_unlock(&hugetlb_lock);
  3038. put_page(page);
  3039. }
  3040. bool is_hugepage_active(struct page *page)
  3041. {
  3042. VM_BUG_ON(!PageHuge(page));
  3043. /*
  3044. * This function can be called for a tail page because the caller,
  3045. * scan_movable_pages, scans through a given pfn-range which typically
  3046. * covers one memory block. In systems using gigantic hugepage (1GB
  3047. * for x86_64,) a hugepage is larger than a memory block, and we don't
  3048. * support migrating such large hugepages for now, so return false
  3049. * when called for tail pages.
  3050. */
  3051. if (PageTail(page))
  3052. return false;
  3053. /*
  3054. * Refcount of a hwpoisoned hugepages is 1, but they are not active,
  3055. * so we should return false for them.
  3056. */
  3057. if (unlikely(PageHWPoison(page)))
  3058. return false;
  3059. return page_count(page) > 0;
  3060. }