page_alloc.c 134 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910
  1. /*
  2. * linux/mm/page_alloc.c
  3. *
  4. * Manages the free list, the system allocates free pages here.
  5. * Note that kmalloc() lives in slab.c
  6. *
  7. * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
  8. * Swap reorganised 29.12.95, Stephen Tweedie
  9. * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
  10. * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
  11. * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
  12. * Zone balancing, Kanoj Sarcar, SGI, Jan 2000
  13. * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
  14. * (lots of bits borrowed from Ingo Molnar & Andrew Morton)
  15. */
  16. #include <linux/stddef.h>
  17. #include <linux/mm.h>
  18. #include <linux/swap.h>
  19. #include <linux/interrupt.h>
  20. #include <linux/pagemap.h>
  21. #include <linux/jiffies.h>
  22. #include <linux/bootmem.h>
  23. #include <linux/compiler.h>
  24. #include <linux/kernel.h>
  25. #include <linux/module.h>
  26. #include <linux/suspend.h>
  27. #include <linux/pagevec.h>
  28. #include <linux/blkdev.h>
  29. #include <linux/slab.h>
  30. #include <linux/oom.h>
  31. #include <linux/notifier.h>
  32. #include <linux/topology.h>
  33. #include <linux/sysctl.h>
  34. #include <linux/cpu.h>
  35. #include <linux/cpuset.h>
  36. #include <linux/memory_hotplug.h>
  37. #include <linux/nodemask.h>
  38. #include <linux/vmalloc.h>
  39. #include <linux/mempolicy.h>
  40. #include <linux/stop_machine.h>
  41. #include <linux/sort.h>
  42. #include <linux/pfn.h>
  43. #include <linux/backing-dev.h>
  44. #include <linux/fault-inject.h>
  45. #include <linux/page-isolation.h>
  46. #include <linux/page_cgroup.h>
  47. #include <linux/debugobjects.h>
  48. #include <linux/kmemleak.h>
  49. #include <asm/tlbflush.h>
  50. #include <asm/div64.h>
  51. #include "internal.h"
  52. /*
  53. * Array of node states.
  54. */
  55. nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
  56. [N_POSSIBLE] = NODE_MASK_ALL,
  57. [N_ONLINE] = { { [0] = 1UL } },
  58. #ifndef CONFIG_NUMA
  59. [N_NORMAL_MEMORY] = { { [0] = 1UL } },
  60. #ifdef CONFIG_HIGHMEM
  61. [N_HIGH_MEMORY] = { { [0] = 1UL } },
  62. #endif
  63. [N_CPU] = { { [0] = 1UL } },
  64. #endif /* NUMA */
  65. };
  66. EXPORT_SYMBOL(node_states);
  67. unsigned long totalram_pages __read_mostly;
  68. unsigned long totalreserve_pages __read_mostly;
  69. unsigned long highest_memmap_pfn __read_mostly;
  70. int percpu_pagelist_fraction;
  71. #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
  72. int pageblock_order __read_mostly;
  73. #endif
  74. static void __free_pages_ok(struct page *page, unsigned int order);
  75. /*
  76. * results with 256, 32 in the lowmem_reserve sysctl:
  77. * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
  78. * 1G machine -> (16M dma, 784M normal, 224M high)
  79. * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
  80. * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
  81. * HIGHMEM allocation will (224M+784M)/256 of ram reserved in ZONE_DMA
  82. *
  83. * TBD: should special case ZONE_DMA32 machines here - in those we normally
  84. * don't need any ZONE_NORMAL reservation
  85. */
  86. int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = {
  87. #ifdef CONFIG_ZONE_DMA
  88. 256,
  89. #endif
  90. #ifdef CONFIG_ZONE_DMA32
  91. 256,
  92. #endif
  93. #ifdef CONFIG_HIGHMEM
  94. 32,
  95. #endif
  96. 32,
  97. };
  98. EXPORT_SYMBOL(totalram_pages);
  99. static char * const zone_names[MAX_NR_ZONES] = {
  100. #ifdef CONFIG_ZONE_DMA
  101. "DMA",
  102. #endif
  103. #ifdef CONFIG_ZONE_DMA32
  104. "DMA32",
  105. #endif
  106. "Normal",
  107. #ifdef CONFIG_HIGHMEM
  108. "HighMem",
  109. #endif
  110. "Movable",
  111. };
  112. int min_free_kbytes = 1024;
  113. unsigned long __meminitdata nr_kernel_pages;
  114. unsigned long __meminitdata nr_all_pages;
  115. static unsigned long __meminitdata dma_reserve;
  116. #ifdef CONFIG_ARCH_POPULATES_NODE_MAP
  117. /*
  118. * MAX_ACTIVE_REGIONS determines the maximum number of distinct
  119. * ranges of memory (RAM) that may be registered with add_active_range().
  120. * Ranges passed to add_active_range() will be merged if possible
  121. * so the number of times add_active_range() can be called is
  122. * related to the number of nodes and the number of holes
  123. */
  124. #ifdef CONFIG_MAX_ACTIVE_REGIONS
  125. /* Allow an architecture to set MAX_ACTIVE_REGIONS to save memory */
  126. #define MAX_ACTIVE_REGIONS CONFIG_MAX_ACTIVE_REGIONS
  127. #else
  128. #if MAX_NUMNODES >= 32
  129. /* If there can be many nodes, allow up to 50 holes per node */
  130. #define MAX_ACTIVE_REGIONS (MAX_NUMNODES*50)
  131. #else
  132. /* By default, allow up to 256 distinct regions */
  133. #define MAX_ACTIVE_REGIONS 256
  134. #endif
  135. #endif
  136. static struct node_active_region __meminitdata early_node_map[MAX_ACTIVE_REGIONS];
  137. static int __meminitdata nr_nodemap_entries;
  138. static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES];
  139. static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES];
  140. static unsigned long __initdata required_kernelcore;
  141. static unsigned long __initdata required_movablecore;
  142. static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES];
  143. /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
  144. int movable_zone;
  145. EXPORT_SYMBOL(movable_zone);
  146. #endif /* CONFIG_ARCH_POPULATES_NODE_MAP */
  147. #if MAX_NUMNODES > 1
  148. int nr_node_ids __read_mostly = MAX_NUMNODES;
  149. int nr_online_nodes __read_mostly = 1;
  150. EXPORT_SYMBOL(nr_node_ids);
  151. EXPORT_SYMBOL(nr_online_nodes);
  152. #endif
  153. int page_group_by_mobility_disabled __read_mostly;
  154. static void set_pageblock_migratetype(struct page *page, int migratetype)
  155. {
  156. if (unlikely(page_group_by_mobility_disabled))
  157. migratetype = MIGRATE_UNMOVABLE;
  158. set_pageblock_flags_group(page, (unsigned long)migratetype,
  159. PB_migrate, PB_migrate_end);
  160. }
  161. bool oom_killer_disabled __read_mostly;
  162. #ifdef CONFIG_DEBUG_VM
  163. static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
  164. {
  165. int ret = 0;
  166. unsigned seq;
  167. unsigned long pfn = page_to_pfn(page);
  168. do {
  169. seq = zone_span_seqbegin(zone);
  170. if (pfn >= zone->zone_start_pfn + zone->spanned_pages)
  171. ret = 1;
  172. else if (pfn < zone->zone_start_pfn)
  173. ret = 1;
  174. } while (zone_span_seqretry(zone, seq));
  175. return ret;
  176. }
  177. static int page_is_consistent(struct zone *zone, struct page *page)
  178. {
  179. if (!pfn_valid_within(page_to_pfn(page)))
  180. return 0;
  181. if (zone != page_zone(page))
  182. return 0;
  183. return 1;
  184. }
  185. /*
  186. * Temporary debugging check for pages not lying within a given zone.
  187. */
  188. static int bad_range(struct zone *zone, struct page *page)
  189. {
  190. if (page_outside_zone_boundaries(zone, page))
  191. return 1;
  192. if (!page_is_consistent(zone, page))
  193. return 1;
  194. return 0;
  195. }
  196. #else
  197. static inline int bad_range(struct zone *zone, struct page *page)
  198. {
  199. return 0;
  200. }
  201. #endif
  202. static void bad_page(struct page *page)
  203. {
  204. static unsigned long resume;
  205. static unsigned long nr_shown;
  206. static unsigned long nr_unshown;
  207. /*
  208. * Allow a burst of 60 reports, then keep quiet for that minute;
  209. * or allow a steady drip of one report per second.
  210. */
  211. if (nr_shown == 60) {
  212. if (time_before(jiffies, resume)) {
  213. nr_unshown++;
  214. goto out;
  215. }
  216. if (nr_unshown) {
  217. printk(KERN_ALERT
  218. "BUG: Bad page state: %lu messages suppressed\n",
  219. nr_unshown);
  220. nr_unshown = 0;
  221. }
  222. nr_shown = 0;
  223. }
  224. if (nr_shown++ == 0)
  225. resume = jiffies + 60 * HZ;
  226. printk(KERN_ALERT "BUG: Bad page state in process %s pfn:%05lx\n",
  227. current->comm, page_to_pfn(page));
  228. printk(KERN_ALERT
  229. "page:%p flags:%p count:%d mapcount:%d mapping:%p index:%lx\n",
  230. page, (void *)page->flags, page_count(page),
  231. page_mapcount(page), page->mapping, page->index);
  232. dump_stack();
  233. out:
  234. /* Leave bad fields for debug, except PageBuddy could make trouble */
  235. __ClearPageBuddy(page);
  236. add_taint(TAINT_BAD_PAGE);
  237. }
  238. /*
  239. * Higher-order pages are called "compound pages". They are structured thusly:
  240. *
  241. * The first PAGE_SIZE page is called the "head page".
  242. *
  243. * The remaining PAGE_SIZE pages are called "tail pages".
  244. *
  245. * All pages have PG_compound set. All pages have their ->private pointing at
  246. * the head page (even the head page has this).
  247. *
  248. * The first tail page's ->lru.next holds the address of the compound page's
  249. * put_page() function. Its ->lru.prev holds the order of allocation.
  250. * This usage means that zero-order pages may not be compound.
  251. */
  252. static void free_compound_page(struct page *page)
  253. {
  254. __free_pages_ok(page, compound_order(page));
  255. }
  256. void prep_compound_page(struct page *page, unsigned long order)
  257. {
  258. int i;
  259. int nr_pages = 1 << order;
  260. set_compound_page_dtor(page, free_compound_page);
  261. set_compound_order(page, order);
  262. __SetPageHead(page);
  263. for (i = 1; i < nr_pages; i++) {
  264. struct page *p = page + i;
  265. __SetPageTail(p);
  266. p->first_page = page;
  267. }
  268. }
  269. static int destroy_compound_page(struct page *page, unsigned long order)
  270. {
  271. int i;
  272. int nr_pages = 1 << order;
  273. int bad = 0;
  274. if (unlikely(compound_order(page) != order) ||
  275. unlikely(!PageHead(page))) {
  276. bad_page(page);
  277. bad++;
  278. }
  279. __ClearPageHead(page);
  280. for (i = 1; i < nr_pages; i++) {
  281. struct page *p = page + i;
  282. if (unlikely(!PageTail(p) || (p->first_page != page))) {
  283. bad_page(page);
  284. bad++;
  285. }
  286. __ClearPageTail(p);
  287. }
  288. return bad;
  289. }
  290. static inline void prep_zero_page(struct page *page, int order, gfp_t gfp_flags)
  291. {
  292. int i;
  293. /*
  294. * clear_highpage() will use KM_USER0, so it's a bug to use __GFP_ZERO
  295. * and __GFP_HIGHMEM from hard or soft interrupt context.
  296. */
  297. VM_BUG_ON((gfp_flags & __GFP_HIGHMEM) && in_interrupt());
  298. for (i = 0; i < (1 << order); i++)
  299. clear_highpage(page + i);
  300. }
  301. static inline void set_page_order(struct page *page, int order)
  302. {
  303. set_page_private(page, order);
  304. __SetPageBuddy(page);
  305. }
  306. static inline void rmv_page_order(struct page *page)
  307. {
  308. __ClearPageBuddy(page);
  309. set_page_private(page, 0);
  310. }
  311. /*
  312. * Locate the struct page for both the matching buddy in our
  313. * pair (buddy1) and the combined O(n+1) page they form (page).
  314. *
  315. * 1) Any buddy B1 will have an order O twin B2 which satisfies
  316. * the following equation:
  317. * B2 = B1 ^ (1 << O)
  318. * For example, if the starting buddy (buddy2) is #8 its order
  319. * 1 buddy is #10:
  320. * B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10
  321. *
  322. * 2) Any buddy B will have an order O+1 parent P which
  323. * satisfies the following equation:
  324. * P = B & ~(1 << O)
  325. *
  326. * Assumption: *_mem_map is contiguous at least up to MAX_ORDER
  327. */
  328. static inline struct page *
  329. __page_find_buddy(struct page *page, unsigned long page_idx, unsigned int order)
  330. {
  331. unsigned long buddy_idx = page_idx ^ (1 << order);
  332. return page + (buddy_idx - page_idx);
  333. }
  334. static inline unsigned long
  335. __find_combined_index(unsigned long page_idx, unsigned int order)
  336. {
  337. return (page_idx & ~(1 << order));
  338. }
  339. /*
  340. * This function checks whether a page is free && is the buddy
  341. * we can do coalesce a page and its buddy if
  342. * (a) the buddy is not in a hole &&
  343. * (b) the buddy is in the buddy system &&
  344. * (c) a page and its buddy have the same order &&
  345. * (d) a page and its buddy are in the same zone.
  346. *
  347. * For recording whether a page is in the buddy system, we use PG_buddy.
  348. * Setting, clearing, and testing PG_buddy is serialized by zone->lock.
  349. *
  350. * For recording page's order, we use page_private(page).
  351. */
  352. static inline int page_is_buddy(struct page *page, struct page *buddy,
  353. int order)
  354. {
  355. if (!pfn_valid_within(page_to_pfn(buddy)))
  356. return 0;
  357. if (page_zone_id(page) != page_zone_id(buddy))
  358. return 0;
  359. if (PageBuddy(buddy) && page_order(buddy) == order) {
  360. VM_BUG_ON(page_count(buddy) != 0);
  361. return 1;
  362. }
  363. return 0;
  364. }
  365. /*
  366. * Freeing function for a buddy system allocator.
  367. *
  368. * The concept of a buddy system is to maintain direct-mapped table
  369. * (containing bit values) for memory blocks of various "orders".
  370. * The bottom level table contains the map for the smallest allocatable
  371. * units of memory (here, pages), and each level above it describes
  372. * pairs of units from the levels below, hence, "buddies".
  373. * At a high level, all that happens here is marking the table entry
  374. * at the bottom level available, and propagating the changes upward
  375. * as necessary, plus some accounting needed to play nicely with other
  376. * parts of the VM system.
  377. * At each level, we keep a list of pages, which are heads of continuous
  378. * free pages of length of (1 << order) and marked with PG_buddy. Page's
  379. * order is recorded in page_private(page) field.
  380. * So when we are allocating or freeing one, we can derive the state of the
  381. * other. That is, if we allocate a small block, and both were
  382. * free, the remainder of the region must be split into blocks.
  383. * If a block is freed, and its buddy is also free, then this
  384. * triggers coalescing into a block of larger size.
  385. *
  386. * -- wli
  387. */
  388. static inline void __free_one_page(struct page *page,
  389. struct zone *zone, unsigned int order,
  390. int migratetype)
  391. {
  392. unsigned long page_idx;
  393. if (unlikely(PageCompound(page)))
  394. if (unlikely(destroy_compound_page(page, order)))
  395. return;
  396. VM_BUG_ON(migratetype == -1);
  397. page_idx = page_to_pfn(page) & ((1 << MAX_ORDER) - 1);
  398. VM_BUG_ON(page_idx & ((1 << order) - 1));
  399. VM_BUG_ON(bad_range(zone, page));
  400. while (order < MAX_ORDER-1) {
  401. unsigned long combined_idx;
  402. struct page *buddy;
  403. buddy = __page_find_buddy(page, page_idx, order);
  404. if (!page_is_buddy(page, buddy, order))
  405. break;
  406. /* Our buddy is free, merge with it and move up one order. */
  407. list_del(&buddy->lru);
  408. zone->free_area[order].nr_free--;
  409. rmv_page_order(buddy);
  410. combined_idx = __find_combined_index(page_idx, order);
  411. page = page + (combined_idx - page_idx);
  412. page_idx = combined_idx;
  413. order++;
  414. }
  415. set_page_order(page, order);
  416. list_add(&page->lru,
  417. &zone->free_area[order].free_list[migratetype]);
  418. zone->free_area[order].nr_free++;
  419. }
  420. #ifdef CONFIG_HAVE_MLOCKED_PAGE_BIT
  421. /*
  422. * free_page_mlock() -- clean up attempts to free and mlocked() page.
  423. * Page should not be on lru, so no need to fix that up.
  424. * free_pages_check() will verify...
  425. */
  426. static inline void free_page_mlock(struct page *page)
  427. {
  428. __ClearPageMlocked(page);
  429. __dec_zone_page_state(page, NR_MLOCK);
  430. __count_vm_event(UNEVICTABLE_MLOCKFREED);
  431. }
  432. #else
  433. static void free_page_mlock(struct page *page) { }
  434. #endif
  435. static inline int free_pages_check(struct page *page)
  436. {
  437. if (unlikely(page_mapcount(page) |
  438. (page->mapping != NULL) |
  439. (atomic_read(&page->_count) != 0) |
  440. (page->flags & PAGE_FLAGS_CHECK_AT_FREE))) {
  441. bad_page(page);
  442. return 1;
  443. }
  444. if (page->flags & PAGE_FLAGS_CHECK_AT_PREP)
  445. page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
  446. return 0;
  447. }
  448. /*
  449. * Frees a list of pages.
  450. * Assumes all pages on list are in same zone, and of same order.
  451. * count is the number of pages to free.
  452. *
  453. * If the zone was previously in an "all pages pinned" state then look to
  454. * see if this freeing clears that state.
  455. *
  456. * And clear the zone's pages_scanned counter, to hold off the "all pages are
  457. * pinned" detection logic.
  458. */
  459. static void free_pages_bulk(struct zone *zone, int count,
  460. struct list_head *list, int order)
  461. {
  462. spin_lock(&zone->lock);
  463. zone_clear_flag(zone, ZONE_ALL_UNRECLAIMABLE);
  464. zone->pages_scanned = 0;
  465. __mod_zone_page_state(zone, NR_FREE_PAGES, count << order);
  466. while (count--) {
  467. struct page *page;
  468. VM_BUG_ON(list_empty(list));
  469. page = list_entry(list->prev, struct page, lru);
  470. /* have to delete it as __free_one_page list manipulates */
  471. list_del(&page->lru);
  472. __free_one_page(page, zone, order, page_private(page));
  473. }
  474. spin_unlock(&zone->lock);
  475. }
  476. static void free_one_page(struct zone *zone, struct page *page, int order,
  477. int migratetype)
  478. {
  479. spin_lock(&zone->lock);
  480. zone_clear_flag(zone, ZONE_ALL_UNRECLAIMABLE);
  481. zone->pages_scanned = 0;
  482. __mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order);
  483. __free_one_page(page, zone, order, migratetype);
  484. spin_unlock(&zone->lock);
  485. }
  486. static void __free_pages_ok(struct page *page, unsigned int order)
  487. {
  488. unsigned long flags;
  489. int i;
  490. int bad = 0;
  491. int clearMlocked = PageMlocked(page);
  492. for (i = 0 ; i < (1 << order) ; ++i)
  493. bad += free_pages_check(page + i);
  494. if (bad)
  495. return;
  496. if (!PageHighMem(page)) {
  497. debug_check_no_locks_freed(page_address(page),PAGE_SIZE<<order);
  498. debug_check_no_obj_freed(page_address(page),
  499. PAGE_SIZE << order);
  500. }
  501. arch_free_page(page, order);
  502. kernel_map_pages(page, 1 << order, 0);
  503. local_irq_save(flags);
  504. if (unlikely(clearMlocked))
  505. free_page_mlock(page);
  506. __count_vm_events(PGFREE, 1 << order);
  507. free_one_page(page_zone(page), page, order,
  508. get_pageblock_migratetype(page));
  509. local_irq_restore(flags);
  510. }
  511. /*
  512. * permit the bootmem allocator to evade page validation on high-order frees
  513. */
  514. void __meminit __free_pages_bootmem(struct page *page, unsigned int order)
  515. {
  516. if (order == 0) {
  517. __ClearPageReserved(page);
  518. set_page_count(page, 0);
  519. set_page_refcounted(page);
  520. __free_page(page);
  521. } else {
  522. int loop;
  523. prefetchw(page);
  524. for (loop = 0; loop < BITS_PER_LONG; loop++) {
  525. struct page *p = &page[loop];
  526. if (loop + 1 < BITS_PER_LONG)
  527. prefetchw(p + 1);
  528. __ClearPageReserved(p);
  529. set_page_count(p, 0);
  530. }
  531. set_page_refcounted(page);
  532. __free_pages(page, order);
  533. }
  534. }
  535. /*
  536. * The order of subdivision here is critical for the IO subsystem.
  537. * Please do not alter this order without good reasons and regression
  538. * testing. Specifically, as large blocks of memory are subdivided,
  539. * the order in which smaller blocks are delivered depends on the order
  540. * they're subdivided in this function. This is the primary factor
  541. * influencing the order in which pages are delivered to the IO
  542. * subsystem according to empirical testing, and this is also justified
  543. * by considering the behavior of a buddy system containing a single
  544. * large block of memory acted on by a series of small allocations.
  545. * This behavior is a critical factor in sglist merging's success.
  546. *
  547. * -- wli
  548. */
  549. static inline void expand(struct zone *zone, struct page *page,
  550. int low, int high, struct free_area *area,
  551. int migratetype)
  552. {
  553. unsigned long size = 1 << high;
  554. while (high > low) {
  555. area--;
  556. high--;
  557. size >>= 1;
  558. VM_BUG_ON(bad_range(zone, &page[size]));
  559. list_add(&page[size].lru, &area->free_list[migratetype]);
  560. area->nr_free++;
  561. set_page_order(&page[size], high);
  562. }
  563. }
  564. /*
  565. * This page is about to be returned from the page allocator
  566. */
  567. static int prep_new_page(struct page *page, int order, gfp_t gfp_flags)
  568. {
  569. if (unlikely(page_mapcount(page) |
  570. (page->mapping != NULL) |
  571. (atomic_read(&page->_count) != 0) |
  572. (page->flags & PAGE_FLAGS_CHECK_AT_PREP))) {
  573. bad_page(page);
  574. return 1;
  575. }
  576. set_page_private(page, 0);
  577. set_page_refcounted(page);
  578. arch_alloc_page(page, order);
  579. kernel_map_pages(page, 1 << order, 1);
  580. if (gfp_flags & __GFP_ZERO)
  581. prep_zero_page(page, order, gfp_flags);
  582. if (order && (gfp_flags & __GFP_COMP))
  583. prep_compound_page(page, order);
  584. return 0;
  585. }
  586. /*
  587. * Go through the free lists for the given migratetype and remove
  588. * the smallest available page from the freelists
  589. */
  590. static inline
  591. struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
  592. int migratetype)
  593. {
  594. unsigned int current_order;
  595. struct free_area * area;
  596. struct page *page;
  597. /* Find a page of the appropriate size in the preferred list */
  598. for (current_order = order; current_order < MAX_ORDER; ++current_order) {
  599. area = &(zone->free_area[current_order]);
  600. if (list_empty(&area->free_list[migratetype]))
  601. continue;
  602. page = list_entry(area->free_list[migratetype].next,
  603. struct page, lru);
  604. list_del(&page->lru);
  605. rmv_page_order(page);
  606. area->nr_free--;
  607. expand(zone, page, order, current_order, area, migratetype);
  608. return page;
  609. }
  610. return NULL;
  611. }
  612. /*
  613. * This array describes the order lists are fallen back to when
  614. * the free lists for the desirable migrate type are depleted
  615. */
  616. static int fallbacks[MIGRATE_TYPES][MIGRATE_TYPES-1] = {
  617. [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE },
  618. [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE },
  619. [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },
  620. [MIGRATE_RESERVE] = { MIGRATE_RESERVE, MIGRATE_RESERVE, MIGRATE_RESERVE }, /* Never used */
  621. };
  622. /*
  623. * Move the free pages in a range to the free lists of the requested type.
  624. * Note that start_page and end_pages are not aligned on a pageblock
  625. * boundary. If alignment is required, use move_freepages_block()
  626. */
  627. static int move_freepages(struct zone *zone,
  628. struct page *start_page, struct page *end_page,
  629. int migratetype)
  630. {
  631. struct page *page;
  632. unsigned long order;
  633. int pages_moved = 0;
  634. #ifndef CONFIG_HOLES_IN_ZONE
  635. /*
  636. * page_zone is not safe to call in this context when
  637. * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant
  638. * anyway as we check zone boundaries in move_freepages_block().
  639. * Remove at a later date when no bug reports exist related to
  640. * grouping pages by mobility
  641. */
  642. BUG_ON(page_zone(start_page) != page_zone(end_page));
  643. #endif
  644. for (page = start_page; page <= end_page;) {
  645. /* Make sure we are not inadvertently changing nodes */
  646. VM_BUG_ON(page_to_nid(page) != zone_to_nid(zone));
  647. if (!pfn_valid_within(page_to_pfn(page))) {
  648. page++;
  649. continue;
  650. }
  651. if (!PageBuddy(page)) {
  652. page++;
  653. continue;
  654. }
  655. order = page_order(page);
  656. list_del(&page->lru);
  657. list_add(&page->lru,
  658. &zone->free_area[order].free_list[migratetype]);
  659. page += 1 << order;
  660. pages_moved += 1 << order;
  661. }
  662. return pages_moved;
  663. }
  664. static int move_freepages_block(struct zone *zone, struct page *page,
  665. int migratetype)
  666. {
  667. unsigned long start_pfn, end_pfn;
  668. struct page *start_page, *end_page;
  669. start_pfn = page_to_pfn(page);
  670. start_pfn = start_pfn & ~(pageblock_nr_pages-1);
  671. start_page = pfn_to_page(start_pfn);
  672. end_page = start_page + pageblock_nr_pages - 1;
  673. end_pfn = start_pfn + pageblock_nr_pages - 1;
  674. /* Do not cross zone boundaries */
  675. if (start_pfn < zone->zone_start_pfn)
  676. start_page = page;
  677. if (end_pfn >= zone->zone_start_pfn + zone->spanned_pages)
  678. return 0;
  679. return move_freepages(zone, start_page, end_page, migratetype);
  680. }
  681. /* Remove an element from the buddy allocator from the fallback list */
  682. static inline struct page *
  683. __rmqueue_fallback(struct zone *zone, int order, int start_migratetype)
  684. {
  685. struct free_area * area;
  686. int current_order;
  687. struct page *page;
  688. int migratetype, i;
  689. /* Find the largest possible block of pages in the other list */
  690. for (current_order = MAX_ORDER-1; current_order >= order;
  691. --current_order) {
  692. for (i = 0; i < MIGRATE_TYPES - 1; i++) {
  693. migratetype = fallbacks[start_migratetype][i];
  694. /* MIGRATE_RESERVE handled later if necessary */
  695. if (migratetype == MIGRATE_RESERVE)
  696. continue;
  697. area = &(zone->free_area[current_order]);
  698. if (list_empty(&area->free_list[migratetype]))
  699. continue;
  700. page = list_entry(area->free_list[migratetype].next,
  701. struct page, lru);
  702. area->nr_free--;
  703. /*
  704. * If breaking a large block of pages, move all free
  705. * pages to the preferred allocation list. If falling
  706. * back for a reclaimable kernel allocation, be more
  707. * agressive about taking ownership of free pages
  708. */
  709. if (unlikely(current_order >= (pageblock_order >> 1)) ||
  710. start_migratetype == MIGRATE_RECLAIMABLE) {
  711. unsigned long pages;
  712. pages = move_freepages_block(zone, page,
  713. start_migratetype);
  714. /* Claim the whole block if over half of it is free */
  715. if (pages >= (1 << (pageblock_order-1)))
  716. set_pageblock_migratetype(page,
  717. start_migratetype);
  718. migratetype = start_migratetype;
  719. }
  720. /* Remove the page from the freelists */
  721. list_del(&page->lru);
  722. rmv_page_order(page);
  723. if (current_order == pageblock_order)
  724. set_pageblock_migratetype(page,
  725. start_migratetype);
  726. expand(zone, page, order, current_order, area, migratetype);
  727. return page;
  728. }
  729. }
  730. return NULL;
  731. }
  732. /*
  733. * Do the hard work of removing an element from the buddy allocator.
  734. * Call me with the zone->lock already held.
  735. */
  736. static struct page *__rmqueue(struct zone *zone, unsigned int order,
  737. int migratetype)
  738. {
  739. struct page *page;
  740. retry_reserve:
  741. page = __rmqueue_smallest(zone, order, migratetype);
  742. if (unlikely(!page) && migratetype != MIGRATE_RESERVE) {
  743. page = __rmqueue_fallback(zone, order, migratetype);
  744. /*
  745. * Use MIGRATE_RESERVE rather than fail an allocation. goto
  746. * is used because __rmqueue_smallest is an inline function
  747. * and we want just one call site
  748. */
  749. if (!page) {
  750. migratetype = MIGRATE_RESERVE;
  751. goto retry_reserve;
  752. }
  753. }
  754. return page;
  755. }
  756. /*
  757. * Obtain a specified number of elements from the buddy allocator, all under
  758. * a single hold of the lock, for efficiency. Add them to the supplied list.
  759. * Returns the number of new pages which were placed at *list.
  760. */
  761. static int rmqueue_bulk(struct zone *zone, unsigned int order,
  762. unsigned long count, struct list_head *list,
  763. int migratetype)
  764. {
  765. int i;
  766. spin_lock(&zone->lock);
  767. for (i = 0; i < count; ++i) {
  768. struct page *page = __rmqueue(zone, order, migratetype);
  769. if (unlikely(page == NULL))
  770. break;
  771. /*
  772. * Split buddy pages returned by expand() are received here
  773. * in physical page order. The page is added to the callers and
  774. * list and the list head then moves forward. From the callers
  775. * perspective, the linked list is ordered by page number in
  776. * some conditions. This is useful for IO devices that can
  777. * merge IO requests if the physical pages are ordered
  778. * properly.
  779. */
  780. list_add(&page->lru, list);
  781. set_page_private(page, migratetype);
  782. list = &page->lru;
  783. }
  784. __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
  785. spin_unlock(&zone->lock);
  786. return i;
  787. }
  788. #ifdef CONFIG_NUMA
  789. /*
  790. * Called from the vmstat counter updater to drain pagesets of this
  791. * currently executing processor on remote nodes after they have
  792. * expired.
  793. *
  794. * Note that this function must be called with the thread pinned to
  795. * a single processor.
  796. */
  797. void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
  798. {
  799. unsigned long flags;
  800. int to_drain;
  801. local_irq_save(flags);
  802. if (pcp->count >= pcp->batch)
  803. to_drain = pcp->batch;
  804. else
  805. to_drain = pcp->count;
  806. free_pages_bulk(zone, to_drain, &pcp->list, 0);
  807. pcp->count -= to_drain;
  808. local_irq_restore(flags);
  809. }
  810. #endif
  811. /*
  812. * Drain pages of the indicated processor.
  813. *
  814. * The processor must either be the current processor and the
  815. * thread pinned to the current processor or a processor that
  816. * is not online.
  817. */
  818. static void drain_pages(unsigned int cpu)
  819. {
  820. unsigned long flags;
  821. struct zone *zone;
  822. for_each_populated_zone(zone) {
  823. struct per_cpu_pageset *pset;
  824. struct per_cpu_pages *pcp;
  825. pset = zone_pcp(zone, cpu);
  826. pcp = &pset->pcp;
  827. local_irq_save(flags);
  828. free_pages_bulk(zone, pcp->count, &pcp->list, 0);
  829. pcp->count = 0;
  830. local_irq_restore(flags);
  831. }
  832. }
  833. /*
  834. * Spill all of this CPU's per-cpu pages back into the buddy allocator.
  835. */
  836. void drain_local_pages(void *arg)
  837. {
  838. drain_pages(smp_processor_id());
  839. }
  840. /*
  841. * Spill all the per-cpu pages from all CPUs back into the buddy allocator
  842. */
  843. void drain_all_pages(void)
  844. {
  845. on_each_cpu(drain_local_pages, NULL, 1);
  846. }
  847. #ifdef CONFIG_HIBERNATION
  848. void mark_free_pages(struct zone *zone)
  849. {
  850. unsigned long pfn, max_zone_pfn;
  851. unsigned long flags;
  852. int order, t;
  853. struct list_head *curr;
  854. if (!zone->spanned_pages)
  855. return;
  856. spin_lock_irqsave(&zone->lock, flags);
  857. max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
  858. for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
  859. if (pfn_valid(pfn)) {
  860. struct page *page = pfn_to_page(pfn);
  861. if (!swsusp_page_is_forbidden(page))
  862. swsusp_unset_page_free(page);
  863. }
  864. for_each_migratetype_order(order, t) {
  865. list_for_each(curr, &zone->free_area[order].free_list[t]) {
  866. unsigned long i;
  867. pfn = page_to_pfn(list_entry(curr, struct page, lru));
  868. for (i = 0; i < (1UL << order); i++)
  869. swsusp_set_page_free(pfn_to_page(pfn + i));
  870. }
  871. }
  872. spin_unlock_irqrestore(&zone->lock, flags);
  873. }
  874. #endif /* CONFIG_PM */
  875. /*
  876. * Free a 0-order page
  877. */
  878. static void free_hot_cold_page(struct page *page, int cold)
  879. {
  880. struct zone *zone = page_zone(page);
  881. struct per_cpu_pages *pcp;
  882. unsigned long flags;
  883. int clearMlocked = PageMlocked(page);
  884. if (PageAnon(page))
  885. page->mapping = NULL;
  886. if (free_pages_check(page))
  887. return;
  888. if (!PageHighMem(page)) {
  889. debug_check_no_locks_freed(page_address(page), PAGE_SIZE);
  890. debug_check_no_obj_freed(page_address(page), PAGE_SIZE);
  891. }
  892. arch_free_page(page, 0);
  893. kernel_map_pages(page, 1, 0);
  894. pcp = &zone_pcp(zone, get_cpu())->pcp;
  895. set_page_private(page, get_pageblock_migratetype(page));
  896. local_irq_save(flags);
  897. if (unlikely(clearMlocked))
  898. free_page_mlock(page);
  899. __count_vm_event(PGFREE);
  900. if (cold)
  901. list_add_tail(&page->lru, &pcp->list);
  902. else
  903. list_add(&page->lru, &pcp->list);
  904. pcp->count++;
  905. if (pcp->count >= pcp->high) {
  906. free_pages_bulk(zone, pcp->batch, &pcp->list, 0);
  907. pcp->count -= pcp->batch;
  908. }
  909. local_irq_restore(flags);
  910. put_cpu();
  911. }
  912. void free_hot_page(struct page *page)
  913. {
  914. free_hot_cold_page(page, 0);
  915. }
  916. void free_cold_page(struct page *page)
  917. {
  918. free_hot_cold_page(page, 1);
  919. }
  920. /*
  921. * split_page takes a non-compound higher-order page, and splits it into
  922. * n (1<<order) sub-pages: page[0..n]
  923. * Each sub-page must be freed individually.
  924. *
  925. * Note: this is probably too low level an operation for use in drivers.
  926. * Please consult with lkml before using this in your driver.
  927. */
  928. void split_page(struct page *page, unsigned int order)
  929. {
  930. int i;
  931. VM_BUG_ON(PageCompound(page));
  932. VM_BUG_ON(!page_count(page));
  933. for (i = 1; i < (1 << order); i++)
  934. set_page_refcounted(page + i);
  935. }
  936. /*
  937. * Really, prep_compound_page() should be called from __rmqueue_bulk(). But
  938. * we cheat by calling it from here, in the order > 0 path. Saves a branch
  939. * or two.
  940. */
  941. static inline
  942. struct page *buffered_rmqueue(struct zone *preferred_zone,
  943. struct zone *zone, int order, gfp_t gfp_flags,
  944. int migratetype)
  945. {
  946. unsigned long flags;
  947. struct page *page;
  948. int cold = !!(gfp_flags & __GFP_COLD);
  949. int cpu;
  950. again:
  951. cpu = get_cpu();
  952. if (likely(order == 0)) {
  953. struct per_cpu_pages *pcp;
  954. pcp = &zone_pcp(zone, cpu)->pcp;
  955. local_irq_save(flags);
  956. if (!pcp->count) {
  957. pcp->count = rmqueue_bulk(zone, 0,
  958. pcp->batch, &pcp->list, migratetype);
  959. if (unlikely(!pcp->count))
  960. goto failed;
  961. }
  962. /* Find a page of the appropriate migrate type */
  963. if (cold) {
  964. list_for_each_entry_reverse(page, &pcp->list, lru)
  965. if (page_private(page) == migratetype)
  966. break;
  967. } else {
  968. list_for_each_entry(page, &pcp->list, lru)
  969. if (page_private(page) == migratetype)
  970. break;
  971. }
  972. /* Allocate more to the pcp list if necessary */
  973. if (unlikely(&page->lru == &pcp->list)) {
  974. pcp->count += rmqueue_bulk(zone, 0,
  975. pcp->batch, &pcp->list, migratetype);
  976. page = list_entry(pcp->list.next, struct page, lru);
  977. }
  978. list_del(&page->lru);
  979. pcp->count--;
  980. } else {
  981. if (unlikely(gfp_flags & __GFP_NOFAIL)) {
  982. /*
  983. * __GFP_NOFAIL is not to be used in new code.
  984. *
  985. * All __GFP_NOFAIL callers should be fixed so that they
  986. * properly detect and handle allocation failures.
  987. *
  988. * We most definitely don't want callers attempting to
  989. * allocate greater than single-page units with
  990. * __GFP_NOFAIL.
  991. */
  992. WARN_ON_ONCE(order > 0);
  993. }
  994. spin_lock_irqsave(&zone->lock, flags);
  995. page = __rmqueue(zone, order, migratetype);
  996. __mod_zone_page_state(zone, NR_FREE_PAGES, -(1 << order));
  997. spin_unlock(&zone->lock);
  998. if (!page)
  999. goto failed;
  1000. }
  1001. __count_zone_vm_events(PGALLOC, zone, 1 << order);
  1002. zone_statistics(preferred_zone, zone);
  1003. local_irq_restore(flags);
  1004. put_cpu();
  1005. VM_BUG_ON(bad_range(zone, page));
  1006. if (prep_new_page(page, order, gfp_flags))
  1007. goto again;
  1008. return page;
  1009. failed:
  1010. local_irq_restore(flags);
  1011. put_cpu();
  1012. return NULL;
  1013. }
  1014. /* The ALLOC_WMARK bits are used as an index to zone->watermark */
  1015. #define ALLOC_WMARK_MIN WMARK_MIN
  1016. #define ALLOC_WMARK_LOW WMARK_LOW
  1017. #define ALLOC_WMARK_HIGH WMARK_HIGH
  1018. #define ALLOC_NO_WATERMARKS 0x04 /* don't check watermarks at all */
  1019. /* Mask to get the watermark bits */
  1020. #define ALLOC_WMARK_MASK (ALLOC_NO_WATERMARKS-1)
  1021. #define ALLOC_HARDER 0x10 /* try to alloc harder */
  1022. #define ALLOC_HIGH 0x20 /* __GFP_HIGH set */
  1023. #define ALLOC_CPUSET 0x40 /* check for correct cpuset */
  1024. #ifdef CONFIG_FAIL_PAGE_ALLOC
  1025. static struct fail_page_alloc_attr {
  1026. struct fault_attr attr;
  1027. u32 ignore_gfp_highmem;
  1028. u32 ignore_gfp_wait;
  1029. u32 min_order;
  1030. #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
  1031. struct dentry *ignore_gfp_highmem_file;
  1032. struct dentry *ignore_gfp_wait_file;
  1033. struct dentry *min_order_file;
  1034. #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
  1035. } fail_page_alloc = {
  1036. .attr = FAULT_ATTR_INITIALIZER,
  1037. .ignore_gfp_wait = 1,
  1038. .ignore_gfp_highmem = 1,
  1039. .min_order = 1,
  1040. };
  1041. static int __init setup_fail_page_alloc(char *str)
  1042. {
  1043. return setup_fault_attr(&fail_page_alloc.attr, str);
  1044. }
  1045. __setup("fail_page_alloc=", setup_fail_page_alloc);
  1046. static int should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
  1047. {
  1048. if (order < fail_page_alloc.min_order)
  1049. return 0;
  1050. if (gfp_mask & __GFP_NOFAIL)
  1051. return 0;
  1052. if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
  1053. return 0;
  1054. if (fail_page_alloc.ignore_gfp_wait && (gfp_mask & __GFP_WAIT))
  1055. return 0;
  1056. return should_fail(&fail_page_alloc.attr, 1 << order);
  1057. }
  1058. #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
  1059. static int __init fail_page_alloc_debugfs(void)
  1060. {
  1061. mode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
  1062. struct dentry *dir;
  1063. int err;
  1064. err = init_fault_attr_dentries(&fail_page_alloc.attr,
  1065. "fail_page_alloc");
  1066. if (err)
  1067. return err;
  1068. dir = fail_page_alloc.attr.dentries.dir;
  1069. fail_page_alloc.ignore_gfp_wait_file =
  1070. debugfs_create_bool("ignore-gfp-wait", mode, dir,
  1071. &fail_page_alloc.ignore_gfp_wait);
  1072. fail_page_alloc.ignore_gfp_highmem_file =
  1073. debugfs_create_bool("ignore-gfp-highmem", mode, dir,
  1074. &fail_page_alloc.ignore_gfp_highmem);
  1075. fail_page_alloc.min_order_file =
  1076. debugfs_create_u32("min-order", mode, dir,
  1077. &fail_page_alloc.min_order);
  1078. if (!fail_page_alloc.ignore_gfp_wait_file ||
  1079. !fail_page_alloc.ignore_gfp_highmem_file ||
  1080. !fail_page_alloc.min_order_file) {
  1081. err = -ENOMEM;
  1082. debugfs_remove(fail_page_alloc.ignore_gfp_wait_file);
  1083. debugfs_remove(fail_page_alloc.ignore_gfp_highmem_file);
  1084. debugfs_remove(fail_page_alloc.min_order_file);
  1085. cleanup_fault_attr_dentries(&fail_page_alloc.attr);
  1086. }
  1087. return err;
  1088. }
  1089. late_initcall(fail_page_alloc_debugfs);
  1090. #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
  1091. #else /* CONFIG_FAIL_PAGE_ALLOC */
  1092. static inline int should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
  1093. {
  1094. return 0;
  1095. }
  1096. #endif /* CONFIG_FAIL_PAGE_ALLOC */
  1097. /*
  1098. * Return 1 if free pages are above 'mark'. This takes into account the order
  1099. * of the allocation.
  1100. */
  1101. int zone_watermark_ok(struct zone *z, int order, unsigned long mark,
  1102. int classzone_idx, int alloc_flags)
  1103. {
  1104. /* free_pages my go negative - that's OK */
  1105. long min = mark;
  1106. long free_pages = zone_page_state(z, NR_FREE_PAGES) - (1 << order) + 1;
  1107. int o;
  1108. if (alloc_flags & ALLOC_HIGH)
  1109. min -= min / 2;
  1110. if (alloc_flags & ALLOC_HARDER)
  1111. min -= min / 4;
  1112. if (free_pages <= min + z->lowmem_reserve[classzone_idx])
  1113. return 0;
  1114. for (o = 0; o < order; o++) {
  1115. /* At the next order, this order's pages become unavailable */
  1116. free_pages -= z->free_area[o].nr_free << o;
  1117. /* Require fewer higher order pages to be free */
  1118. min >>= 1;
  1119. if (free_pages <= min)
  1120. return 0;
  1121. }
  1122. return 1;
  1123. }
  1124. #ifdef CONFIG_NUMA
  1125. /*
  1126. * zlc_setup - Setup for "zonelist cache". Uses cached zone data to
  1127. * skip over zones that are not allowed by the cpuset, or that have
  1128. * been recently (in last second) found to be nearly full. See further
  1129. * comments in mmzone.h. Reduces cache footprint of zonelist scans
  1130. * that have to skip over a lot of full or unallowed zones.
  1131. *
  1132. * If the zonelist cache is present in the passed in zonelist, then
  1133. * returns a pointer to the allowed node mask (either the current
  1134. * tasks mems_allowed, or node_states[N_HIGH_MEMORY].)
  1135. *
  1136. * If the zonelist cache is not available for this zonelist, does
  1137. * nothing and returns NULL.
  1138. *
  1139. * If the fullzones BITMAP in the zonelist cache is stale (more than
  1140. * a second since last zap'd) then we zap it out (clear its bits.)
  1141. *
  1142. * We hold off even calling zlc_setup, until after we've checked the
  1143. * first zone in the zonelist, on the theory that most allocations will
  1144. * be satisfied from that first zone, so best to examine that zone as
  1145. * quickly as we can.
  1146. */
  1147. static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
  1148. {
  1149. struct zonelist_cache *zlc; /* cached zonelist speedup info */
  1150. nodemask_t *allowednodes; /* zonelist_cache approximation */
  1151. zlc = zonelist->zlcache_ptr;
  1152. if (!zlc)
  1153. return NULL;
  1154. if (time_after(jiffies, zlc->last_full_zap + HZ)) {
  1155. bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
  1156. zlc->last_full_zap = jiffies;
  1157. }
  1158. allowednodes = !in_interrupt() && (alloc_flags & ALLOC_CPUSET) ?
  1159. &cpuset_current_mems_allowed :
  1160. &node_states[N_HIGH_MEMORY];
  1161. return allowednodes;
  1162. }
  1163. /*
  1164. * Given 'z' scanning a zonelist, run a couple of quick checks to see
  1165. * if it is worth looking at further for free memory:
  1166. * 1) Check that the zone isn't thought to be full (doesn't have its
  1167. * bit set in the zonelist_cache fullzones BITMAP).
  1168. * 2) Check that the zones node (obtained from the zonelist_cache
  1169. * z_to_n[] mapping) is allowed in the passed in allowednodes mask.
  1170. * Return true (non-zero) if zone is worth looking at further, or
  1171. * else return false (zero) if it is not.
  1172. *
  1173. * This check -ignores- the distinction between various watermarks,
  1174. * such as GFP_HIGH, GFP_ATOMIC, PF_MEMALLOC, ... If a zone is
  1175. * found to be full for any variation of these watermarks, it will
  1176. * be considered full for up to one second by all requests, unless
  1177. * we are so low on memory on all allowed nodes that we are forced
  1178. * into the second scan of the zonelist.
  1179. *
  1180. * In the second scan we ignore this zonelist cache and exactly
  1181. * apply the watermarks to all zones, even it is slower to do so.
  1182. * We are low on memory in the second scan, and should leave no stone
  1183. * unturned looking for a free page.
  1184. */
  1185. static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z,
  1186. nodemask_t *allowednodes)
  1187. {
  1188. struct zonelist_cache *zlc; /* cached zonelist speedup info */
  1189. int i; /* index of *z in zonelist zones */
  1190. int n; /* node that zone *z is on */
  1191. zlc = zonelist->zlcache_ptr;
  1192. if (!zlc)
  1193. return 1;
  1194. i = z - zonelist->_zonerefs;
  1195. n = zlc->z_to_n[i];
  1196. /* This zone is worth trying if it is allowed but not full */
  1197. return node_isset(n, *allowednodes) && !test_bit(i, zlc->fullzones);
  1198. }
  1199. /*
  1200. * Given 'z' scanning a zonelist, set the corresponding bit in
  1201. * zlc->fullzones, so that subsequent attempts to allocate a page
  1202. * from that zone don't waste time re-examining it.
  1203. */
  1204. static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z)
  1205. {
  1206. struct zonelist_cache *zlc; /* cached zonelist speedup info */
  1207. int i; /* index of *z in zonelist zones */
  1208. zlc = zonelist->zlcache_ptr;
  1209. if (!zlc)
  1210. return;
  1211. i = z - zonelist->_zonerefs;
  1212. set_bit(i, zlc->fullzones);
  1213. }
  1214. #else /* CONFIG_NUMA */
  1215. static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
  1216. {
  1217. return NULL;
  1218. }
  1219. static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z,
  1220. nodemask_t *allowednodes)
  1221. {
  1222. return 1;
  1223. }
  1224. static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z)
  1225. {
  1226. }
  1227. #endif /* CONFIG_NUMA */
  1228. /*
  1229. * get_page_from_freelist goes through the zonelist trying to allocate
  1230. * a page.
  1231. */
  1232. static struct page *
  1233. get_page_from_freelist(gfp_t gfp_mask, nodemask_t *nodemask, unsigned int order,
  1234. struct zonelist *zonelist, int high_zoneidx, int alloc_flags,
  1235. struct zone *preferred_zone, int migratetype)
  1236. {
  1237. struct zoneref *z;
  1238. struct page *page = NULL;
  1239. int classzone_idx;
  1240. struct zone *zone;
  1241. nodemask_t *allowednodes = NULL;/* zonelist_cache approximation */
  1242. int zlc_active = 0; /* set if using zonelist_cache */
  1243. int did_zlc_setup = 0; /* just call zlc_setup() one time */
  1244. classzone_idx = zone_idx(preferred_zone);
  1245. zonelist_scan:
  1246. /*
  1247. * Scan zonelist, looking for a zone with enough free.
  1248. * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
  1249. */
  1250. for_each_zone_zonelist_nodemask(zone, z, zonelist,
  1251. high_zoneidx, nodemask) {
  1252. if (NUMA_BUILD && zlc_active &&
  1253. !zlc_zone_worth_trying(zonelist, z, allowednodes))
  1254. continue;
  1255. if ((alloc_flags & ALLOC_CPUSET) &&
  1256. !cpuset_zone_allowed_softwall(zone, gfp_mask))
  1257. goto try_next_zone;
  1258. BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
  1259. if (!(alloc_flags & ALLOC_NO_WATERMARKS)) {
  1260. unsigned long mark;
  1261. mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK];
  1262. if (!zone_watermark_ok(zone, order, mark,
  1263. classzone_idx, alloc_flags)) {
  1264. if (!zone_reclaim_mode ||
  1265. !zone_reclaim(zone, gfp_mask, order))
  1266. goto this_zone_full;
  1267. }
  1268. }
  1269. page = buffered_rmqueue(preferred_zone, zone, order,
  1270. gfp_mask, migratetype);
  1271. if (page)
  1272. break;
  1273. this_zone_full:
  1274. if (NUMA_BUILD)
  1275. zlc_mark_zone_full(zonelist, z);
  1276. try_next_zone:
  1277. if (NUMA_BUILD && !did_zlc_setup && nr_online_nodes > 1) {
  1278. /*
  1279. * we do zlc_setup after the first zone is tried but only
  1280. * if there are multiple nodes make it worthwhile
  1281. */
  1282. allowednodes = zlc_setup(zonelist, alloc_flags);
  1283. zlc_active = 1;
  1284. did_zlc_setup = 1;
  1285. }
  1286. }
  1287. if (unlikely(NUMA_BUILD && page == NULL && zlc_active)) {
  1288. /* Disable zlc cache for second zonelist scan */
  1289. zlc_active = 0;
  1290. goto zonelist_scan;
  1291. }
  1292. return page;
  1293. }
  1294. static inline int
  1295. should_alloc_retry(gfp_t gfp_mask, unsigned int order,
  1296. unsigned long pages_reclaimed)
  1297. {
  1298. /* Do not loop if specifically requested */
  1299. if (gfp_mask & __GFP_NORETRY)
  1300. return 0;
  1301. /*
  1302. * In this implementation, order <= PAGE_ALLOC_COSTLY_ORDER
  1303. * means __GFP_NOFAIL, but that may not be true in other
  1304. * implementations.
  1305. */
  1306. if (order <= PAGE_ALLOC_COSTLY_ORDER)
  1307. return 1;
  1308. /*
  1309. * For order > PAGE_ALLOC_COSTLY_ORDER, if __GFP_REPEAT is
  1310. * specified, then we retry until we no longer reclaim any pages
  1311. * (above), or we've reclaimed an order of pages at least as
  1312. * large as the allocation's order. In both cases, if the
  1313. * allocation still fails, we stop retrying.
  1314. */
  1315. if (gfp_mask & __GFP_REPEAT && pages_reclaimed < (1 << order))
  1316. return 1;
  1317. /*
  1318. * Don't let big-order allocations loop unless the caller
  1319. * explicitly requests that.
  1320. */
  1321. if (gfp_mask & __GFP_NOFAIL)
  1322. return 1;
  1323. return 0;
  1324. }
  1325. static inline struct page *
  1326. __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
  1327. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1328. nodemask_t *nodemask, struct zone *preferred_zone,
  1329. int migratetype)
  1330. {
  1331. struct page *page;
  1332. /* Acquire the OOM killer lock for the zones in zonelist */
  1333. if (!try_set_zone_oom(zonelist, gfp_mask)) {
  1334. schedule_timeout_uninterruptible(1);
  1335. return NULL;
  1336. }
  1337. /*
  1338. * Go through the zonelist yet one more time, keep very high watermark
  1339. * here, this is only to catch a parallel oom killing, we must fail if
  1340. * we're still under heavy pressure.
  1341. */
  1342. page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask,
  1343. order, zonelist, high_zoneidx,
  1344. ALLOC_WMARK_HIGH|ALLOC_CPUSET,
  1345. preferred_zone, migratetype);
  1346. if (page)
  1347. goto out;
  1348. /* The OOM killer will not help higher order allocs */
  1349. if (order > PAGE_ALLOC_COSTLY_ORDER)
  1350. goto out;
  1351. /* Exhausted what can be done so it's blamo time */
  1352. out_of_memory(zonelist, gfp_mask, order);
  1353. out:
  1354. clear_zonelist_oom(zonelist, gfp_mask);
  1355. return page;
  1356. }
  1357. /* The really slow allocator path where we enter direct reclaim */
  1358. static inline struct page *
  1359. __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
  1360. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1361. nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
  1362. int migratetype, unsigned long *did_some_progress)
  1363. {
  1364. struct page *page = NULL;
  1365. struct reclaim_state reclaim_state;
  1366. struct task_struct *p = current;
  1367. cond_resched();
  1368. /* We now go into synchronous reclaim */
  1369. cpuset_memory_pressure_bump();
  1370. /*
  1371. * The task's cpuset might have expanded its set of allowable nodes
  1372. */
  1373. p->flags |= PF_MEMALLOC;
  1374. lockdep_set_current_reclaim_state(gfp_mask);
  1375. reclaim_state.reclaimed_slab = 0;
  1376. p->reclaim_state = &reclaim_state;
  1377. *did_some_progress = try_to_free_pages(zonelist, order, gfp_mask, nodemask);
  1378. p->reclaim_state = NULL;
  1379. lockdep_clear_current_reclaim_state();
  1380. p->flags &= ~PF_MEMALLOC;
  1381. cond_resched();
  1382. if (order != 0)
  1383. drain_all_pages();
  1384. if (likely(*did_some_progress))
  1385. page = get_page_from_freelist(gfp_mask, nodemask, order,
  1386. zonelist, high_zoneidx,
  1387. alloc_flags, preferred_zone,
  1388. migratetype);
  1389. return page;
  1390. }
  1391. /*
  1392. * This is called in the allocator slow-path if the allocation request is of
  1393. * sufficient urgency to ignore watermarks and take other desperate measures
  1394. */
  1395. static inline struct page *
  1396. __alloc_pages_high_priority(gfp_t gfp_mask, unsigned int order,
  1397. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1398. nodemask_t *nodemask, struct zone *preferred_zone,
  1399. int migratetype)
  1400. {
  1401. struct page *page;
  1402. do {
  1403. page = get_page_from_freelist(gfp_mask, nodemask, order,
  1404. zonelist, high_zoneidx, ALLOC_NO_WATERMARKS,
  1405. preferred_zone, migratetype);
  1406. if (!page && gfp_mask & __GFP_NOFAIL)
  1407. congestion_wait(WRITE, HZ/50);
  1408. } while (!page && (gfp_mask & __GFP_NOFAIL));
  1409. return page;
  1410. }
  1411. static inline
  1412. void wake_all_kswapd(unsigned int order, struct zonelist *zonelist,
  1413. enum zone_type high_zoneidx)
  1414. {
  1415. struct zoneref *z;
  1416. struct zone *zone;
  1417. for_each_zone_zonelist(zone, z, zonelist, high_zoneidx)
  1418. wakeup_kswapd(zone, order);
  1419. }
  1420. static inline int
  1421. gfp_to_alloc_flags(gfp_t gfp_mask)
  1422. {
  1423. struct task_struct *p = current;
  1424. int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
  1425. const gfp_t wait = gfp_mask & __GFP_WAIT;
  1426. /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */
  1427. BUILD_BUG_ON(__GFP_HIGH != ALLOC_HIGH);
  1428. /*
  1429. * The caller may dip into page reserves a bit more if the caller
  1430. * cannot run direct reclaim, or if the caller has realtime scheduling
  1431. * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will
  1432. * set both ALLOC_HARDER (!wait) and ALLOC_HIGH (__GFP_HIGH).
  1433. */
  1434. alloc_flags |= (gfp_mask & __GFP_HIGH);
  1435. if (!wait) {
  1436. alloc_flags |= ALLOC_HARDER;
  1437. /*
  1438. * Ignore cpuset if GFP_ATOMIC (!wait) rather than fail alloc.
  1439. * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
  1440. */
  1441. alloc_flags &= ~ALLOC_CPUSET;
  1442. } else if (unlikely(rt_task(p)))
  1443. alloc_flags |= ALLOC_HARDER;
  1444. if (likely(!(gfp_mask & __GFP_NOMEMALLOC))) {
  1445. if (!in_interrupt() &&
  1446. ((p->flags & PF_MEMALLOC) ||
  1447. unlikely(test_thread_flag(TIF_MEMDIE))))
  1448. alloc_flags |= ALLOC_NO_WATERMARKS;
  1449. }
  1450. return alloc_flags;
  1451. }
  1452. static inline struct page *
  1453. __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
  1454. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1455. nodemask_t *nodemask, struct zone *preferred_zone,
  1456. int migratetype)
  1457. {
  1458. const gfp_t wait = gfp_mask & __GFP_WAIT;
  1459. struct page *page = NULL;
  1460. int alloc_flags;
  1461. unsigned long pages_reclaimed = 0;
  1462. unsigned long did_some_progress;
  1463. struct task_struct *p = current;
  1464. /*
  1465. * In the slowpath, we sanity check order to avoid ever trying to
  1466. * reclaim >= MAX_ORDER areas which will never succeed. Callers may
  1467. * be using allocators in order of preference for an area that is
  1468. * too large.
  1469. */
  1470. if (WARN_ON_ONCE(order >= MAX_ORDER))
  1471. return NULL;
  1472. /*
  1473. * GFP_THISNODE (meaning __GFP_THISNODE, __GFP_NORETRY and
  1474. * __GFP_NOWARN set) should not cause reclaim since the subsystem
  1475. * (f.e. slab) using GFP_THISNODE may choose to trigger reclaim
  1476. * using a larger set of nodes after it has established that the
  1477. * allowed per node queues are empty and that nodes are
  1478. * over allocated.
  1479. */
  1480. if (NUMA_BUILD && (gfp_mask & GFP_THISNODE) == GFP_THISNODE)
  1481. goto nopage;
  1482. wake_all_kswapd(order, zonelist, high_zoneidx);
  1483. /*
  1484. * OK, we're below the kswapd watermark and have kicked background
  1485. * reclaim. Now things get more complex, so set up alloc_flags according
  1486. * to how we want to proceed.
  1487. */
  1488. alloc_flags = gfp_to_alloc_flags(gfp_mask);
  1489. restart:
  1490. /* This is the last chance, in general, before the goto nopage. */
  1491. page = get_page_from_freelist(gfp_mask, nodemask, order, zonelist,
  1492. high_zoneidx, alloc_flags & ~ALLOC_NO_WATERMARKS,
  1493. preferred_zone, migratetype);
  1494. if (page)
  1495. goto got_pg;
  1496. rebalance:
  1497. /* Allocate without watermarks if the context allows */
  1498. if (alloc_flags & ALLOC_NO_WATERMARKS) {
  1499. page = __alloc_pages_high_priority(gfp_mask, order,
  1500. zonelist, high_zoneidx, nodemask,
  1501. preferred_zone, migratetype);
  1502. if (page)
  1503. goto got_pg;
  1504. }
  1505. /* Atomic allocations - we can't balance anything */
  1506. if (!wait)
  1507. goto nopage;
  1508. /* Avoid recursion of direct reclaim */
  1509. if (p->flags & PF_MEMALLOC)
  1510. goto nopage;
  1511. /* Try direct reclaim and then allocating */
  1512. page = __alloc_pages_direct_reclaim(gfp_mask, order,
  1513. zonelist, high_zoneidx,
  1514. nodemask,
  1515. alloc_flags, preferred_zone,
  1516. migratetype, &did_some_progress);
  1517. if (page)
  1518. goto got_pg;
  1519. /*
  1520. * If we failed to make any progress reclaiming, then we are
  1521. * running out of options and have to consider going OOM
  1522. */
  1523. if (!did_some_progress) {
  1524. if ((gfp_mask & __GFP_FS) && !(gfp_mask & __GFP_NORETRY)) {
  1525. if (oom_killer_disabled)
  1526. goto nopage;
  1527. page = __alloc_pages_may_oom(gfp_mask, order,
  1528. zonelist, high_zoneidx,
  1529. nodemask, preferred_zone,
  1530. migratetype);
  1531. if (page)
  1532. goto got_pg;
  1533. /*
  1534. * The OOM killer does not trigger for high-order allocations
  1535. * but if no progress is being made, there are no other
  1536. * options and retrying is unlikely to help
  1537. */
  1538. if (order > PAGE_ALLOC_COSTLY_ORDER)
  1539. goto nopage;
  1540. goto restart;
  1541. }
  1542. }
  1543. /* Check if we should retry the allocation */
  1544. pages_reclaimed += did_some_progress;
  1545. if (should_alloc_retry(gfp_mask, order, pages_reclaimed)) {
  1546. /* Wait for some write requests to complete then retry */
  1547. congestion_wait(WRITE, HZ/50);
  1548. goto rebalance;
  1549. }
  1550. nopage:
  1551. if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit()) {
  1552. printk(KERN_WARNING "%s: page allocation failure."
  1553. " order:%d, mode:0x%x\n",
  1554. p->comm, order, gfp_mask);
  1555. dump_stack();
  1556. show_mem();
  1557. }
  1558. got_pg:
  1559. return page;
  1560. }
  1561. /*
  1562. * This is the 'heart' of the zoned buddy allocator.
  1563. */
  1564. struct page *
  1565. __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
  1566. struct zonelist *zonelist, nodemask_t *nodemask)
  1567. {
  1568. enum zone_type high_zoneidx = gfp_zone(gfp_mask);
  1569. struct zone *preferred_zone;
  1570. struct page *page;
  1571. int migratetype = allocflags_to_migratetype(gfp_mask);
  1572. lockdep_trace_alloc(gfp_mask);
  1573. might_sleep_if(gfp_mask & __GFP_WAIT);
  1574. if (should_fail_alloc_page(gfp_mask, order))
  1575. return NULL;
  1576. /*
  1577. * Check the zones suitable for the gfp_mask contain at least one
  1578. * valid zone. It's possible to have an empty zonelist as a result
  1579. * of GFP_THISNODE and a memoryless node
  1580. */
  1581. if (unlikely(!zonelist->_zonerefs->zone))
  1582. return NULL;
  1583. /* The preferred zone is used for statistics later */
  1584. first_zones_zonelist(zonelist, high_zoneidx, nodemask, &preferred_zone);
  1585. if (!preferred_zone)
  1586. return NULL;
  1587. /* First allocation attempt */
  1588. page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask, order,
  1589. zonelist, high_zoneidx, ALLOC_WMARK_LOW|ALLOC_CPUSET,
  1590. preferred_zone, migratetype);
  1591. if (unlikely(!page))
  1592. page = __alloc_pages_slowpath(gfp_mask, order,
  1593. zonelist, high_zoneidx, nodemask,
  1594. preferred_zone, migratetype);
  1595. return page;
  1596. }
  1597. EXPORT_SYMBOL(__alloc_pages_nodemask);
  1598. /*
  1599. * Common helper functions.
  1600. */
  1601. unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
  1602. {
  1603. struct page * page;
  1604. page = alloc_pages(gfp_mask, order);
  1605. if (!page)
  1606. return 0;
  1607. return (unsigned long) page_address(page);
  1608. }
  1609. EXPORT_SYMBOL(__get_free_pages);
  1610. unsigned long get_zeroed_page(gfp_t gfp_mask)
  1611. {
  1612. struct page * page;
  1613. /*
  1614. * get_zeroed_page() returns a 32-bit address, which cannot represent
  1615. * a highmem page
  1616. */
  1617. VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0);
  1618. page = alloc_pages(gfp_mask | __GFP_ZERO, 0);
  1619. if (page)
  1620. return (unsigned long) page_address(page);
  1621. return 0;
  1622. }
  1623. EXPORT_SYMBOL(get_zeroed_page);
  1624. void __pagevec_free(struct pagevec *pvec)
  1625. {
  1626. int i = pagevec_count(pvec);
  1627. while (--i >= 0)
  1628. free_hot_cold_page(pvec->pages[i], pvec->cold);
  1629. }
  1630. void __free_pages(struct page *page, unsigned int order)
  1631. {
  1632. if (put_page_testzero(page)) {
  1633. if (order == 0)
  1634. free_hot_page(page);
  1635. else
  1636. __free_pages_ok(page, order);
  1637. }
  1638. }
  1639. EXPORT_SYMBOL(__free_pages);
  1640. void free_pages(unsigned long addr, unsigned int order)
  1641. {
  1642. if (addr != 0) {
  1643. VM_BUG_ON(!virt_addr_valid((void *)addr));
  1644. __free_pages(virt_to_page((void *)addr), order);
  1645. }
  1646. }
  1647. EXPORT_SYMBOL(free_pages);
  1648. /**
  1649. * alloc_pages_exact - allocate an exact number physically-contiguous pages.
  1650. * @size: the number of bytes to allocate
  1651. * @gfp_mask: GFP flags for the allocation
  1652. *
  1653. * This function is similar to alloc_pages(), except that it allocates the
  1654. * minimum number of pages to satisfy the request. alloc_pages() can only
  1655. * allocate memory in power-of-two pages.
  1656. *
  1657. * This function is also limited by MAX_ORDER.
  1658. *
  1659. * Memory allocated by this function must be released by free_pages_exact().
  1660. */
  1661. void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
  1662. {
  1663. unsigned int order = get_order(size);
  1664. unsigned long addr;
  1665. addr = __get_free_pages(gfp_mask, order);
  1666. if (addr) {
  1667. unsigned long alloc_end = addr + (PAGE_SIZE << order);
  1668. unsigned long used = addr + PAGE_ALIGN(size);
  1669. split_page(virt_to_page(addr), order);
  1670. while (used < alloc_end) {
  1671. free_page(used);
  1672. used += PAGE_SIZE;
  1673. }
  1674. }
  1675. return (void *)addr;
  1676. }
  1677. EXPORT_SYMBOL(alloc_pages_exact);
  1678. /**
  1679. * free_pages_exact - release memory allocated via alloc_pages_exact()
  1680. * @virt: the value returned by alloc_pages_exact.
  1681. * @size: size of allocation, same value as passed to alloc_pages_exact().
  1682. *
  1683. * Release the memory allocated by a previous call to alloc_pages_exact.
  1684. */
  1685. void free_pages_exact(void *virt, size_t size)
  1686. {
  1687. unsigned long addr = (unsigned long)virt;
  1688. unsigned long end = addr + PAGE_ALIGN(size);
  1689. while (addr < end) {
  1690. free_page(addr);
  1691. addr += PAGE_SIZE;
  1692. }
  1693. }
  1694. EXPORT_SYMBOL(free_pages_exact);
  1695. static unsigned int nr_free_zone_pages(int offset)
  1696. {
  1697. struct zoneref *z;
  1698. struct zone *zone;
  1699. /* Just pick one node, since fallback list is circular */
  1700. unsigned int sum = 0;
  1701. struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
  1702. for_each_zone_zonelist(zone, z, zonelist, offset) {
  1703. unsigned long size = zone->present_pages;
  1704. unsigned long high = high_wmark_pages(zone);
  1705. if (size > high)
  1706. sum += size - high;
  1707. }
  1708. return sum;
  1709. }
  1710. /*
  1711. * Amount of free RAM allocatable within ZONE_DMA and ZONE_NORMAL
  1712. */
  1713. unsigned int nr_free_buffer_pages(void)
  1714. {
  1715. return nr_free_zone_pages(gfp_zone(GFP_USER));
  1716. }
  1717. EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
  1718. /*
  1719. * Amount of free RAM allocatable within all zones
  1720. */
  1721. unsigned int nr_free_pagecache_pages(void)
  1722. {
  1723. return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
  1724. }
  1725. static inline void show_node(struct zone *zone)
  1726. {
  1727. if (NUMA_BUILD)
  1728. printk("Node %d ", zone_to_nid(zone));
  1729. }
  1730. void si_meminfo(struct sysinfo *val)
  1731. {
  1732. val->totalram = totalram_pages;
  1733. val->sharedram = 0;
  1734. val->freeram = global_page_state(NR_FREE_PAGES);
  1735. val->bufferram = nr_blockdev_pages();
  1736. val->totalhigh = totalhigh_pages;
  1737. val->freehigh = nr_free_highpages();
  1738. val->mem_unit = PAGE_SIZE;
  1739. }
  1740. EXPORT_SYMBOL(si_meminfo);
  1741. #ifdef CONFIG_NUMA
  1742. void si_meminfo_node(struct sysinfo *val, int nid)
  1743. {
  1744. pg_data_t *pgdat = NODE_DATA(nid);
  1745. val->totalram = pgdat->node_present_pages;
  1746. val->freeram = node_page_state(nid, NR_FREE_PAGES);
  1747. #ifdef CONFIG_HIGHMEM
  1748. val->totalhigh = pgdat->node_zones[ZONE_HIGHMEM].present_pages;
  1749. val->freehigh = zone_page_state(&pgdat->node_zones[ZONE_HIGHMEM],
  1750. NR_FREE_PAGES);
  1751. #else
  1752. val->totalhigh = 0;
  1753. val->freehigh = 0;
  1754. #endif
  1755. val->mem_unit = PAGE_SIZE;
  1756. }
  1757. #endif
  1758. #define K(x) ((x) << (PAGE_SHIFT-10))
  1759. /*
  1760. * Show free area list (used inside shift_scroll-lock stuff)
  1761. * We also calculate the percentage fragmentation. We do this by counting the
  1762. * memory on each free list with the exception of the first item on the list.
  1763. */
  1764. void show_free_areas(void)
  1765. {
  1766. int cpu;
  1767. struct zone *zone;
  1768. for_each_populated_zone(zone) {
  1769. show_node(zone);
  1770. printk("%s per-cpu:\n", zone->name);
  1771. for_each_online_cpu(cpu) {
  1772. struct per_cpu_pageset *pageset;
  1773. pageset = zone_pcp(zone, cpu);
  1774. printk("CPU %4d: hi:%5d, btch:%4d usd:%4d\n",
  1775. cpu, pageset->pcp.high,
  1776. pageset->pcp.batch, pageset->pcp.count);
  1777. }
  1778. }
  1779. printk("Active_anon:%lu active_file:%lu inactive_anon:%lu\n"
  1780. " inactive_file:%lu"
  1781. //TODO: check/adjust line lengths
  1782. #ifdef CONFIG_UNEVICTABLE_LRU
  1783. " unevictable:%lu"
  1784. #endif
  1785. " dirty:%lu writeback:%lu unstable:%lu\n"
  1786. " free:%lu slab:%lu mapped:%lu pagetables:%lu bounce:%lu\n",
  1787. global_page_state(NR_ACTIVE_ANON),
  1788. global_page_state(NR_ACTIVE_FILE),
  1789. global_page_state(NR_INACTIVE_ANON),
  1790. global_page_state(NR_INACTIVE_FILE),
  1791. #ifdef CONFIG_UNEVICTABLE_LRU
  1792. global_page_state(NR_UNEVICTABLE),
  1793. #endif
  1794. global_page_state(NR_FILE_DIRTY),
  1795. global_page_state(NR_WRITEBACK),
  1796. global_page_state(NR_UNSTABLE_NFS),
  1797. global_page_state(NR_FREE_PAGES),
  1798. global_page_state(NR_SLAB_RECLAIMABLE) +
  1799. global_page_state(NR_SLAB_UNRECLAIMABLE),
  1800. global_page_state(NR_FILE_MAPPED),
  1801. global_page_state(NR_PAGETABLE),
  1802. global_page_state(NR_BOUNCE));
  1803. for_each_populated_zone(zone) {
  1804. int i;
  1805. show_node(zone);
  1806. printk("%s"
  1807. " free:%lukB"
  1808. " min:%lukB"
  1809. " low:%lukB"
  1810. " high:%lukB"
  1811. " active_anon:%lukB"
  1812. " inactive_anon:%lukB"
  1813. " active_file:%lukB"
  1814. " inactive_file:%lukB"
  1815. #ifdef CONFIG_UNEVICTABLE_LRU
  1816. " unevictable:%lukB"
  1817. #endif
  1818. " present:%lukB"
  1819. " pages_scanned:%lu"
  1820. " all_unreclaimable? %s"
  1821. "\n",
  1822. zone->name,
  1823. K(zone_page_state(zone, NR_FREE_PAGES)),
  1824. K(min_wmark_pages(zone)),
  1825. K(low_wmark_pages(zone)),
  1826. K(high_wmark_pages(zone)),
  1827. K(zone_page_state(zone, NR_ACTIVE_ANON)),
  1828. K(zone_page_state(zone, NR_INACTIVE_ANON)),
  1829. K(zone_page_state(zone, NR_ACTIVE_FILE)),
  1830. K(zone_page_state(zone, NR_INACTIVE_FILE)),
  1831. #ifdef CONFIG_UNEVICTABLE_LRU
  1832. K(zone_page_state(zone, NR_UNEVICTABLE)),
  1833. #endif
  1834. K(zone->present_pages),
  1835. zone->pages_scanned,
  1836. (zone_is_all_unreclaimable(zone) ? "yes" : "no")
  1837. );
  1838. printk("lowmem_reserve[]:");
  1839. for (i = 0; i < MAX_NR_ZONES; i++)
  1840. printk(" %lu", zone->lowmem_reserve[i]);
  1841. printk("\n");
  1842. }
  1843. for_each_populated_zone(zone) {
  1844. unsigned long nr[MAX_ORDER], flags, order, total = 0;
  1845. show_node(zone);
  1846. printk("%s: ", zone->name);
  1847. spin_lock_irqsave(&zone->lock, flags);
  1848. for (order = 0; order < MAX_ORDER; order++) {
  1849. nr[order] = zone->free_area[order].nr_free;
  1850. total += nr[order] << order;
  1851. }
  1852. spin_unlock_irqrestore(&zone->lock, flags);
  1853. for (order = 0; order < MAX_ORDER; order++)
  1854. printk("%lu*%lukB ", nr[order], K(1UL) << order);
  1855. printk("= %lukB\n", K(total));
  1856. }
  1857. printk("%ld total pagecache pages\n", global_page_state(NR_FILE_PAGES));
  1858. show_swap_cache_info();
  1859. }
  1860. static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
  1861. {
  1862. zoneref->zone = zone;
  1863. zoneref->zone_idx = zone_idx(zone);
  1864. }
  1865. /*
  1866. * Builds allocation fallback zone lists.
  1867. *
  1868. * Add all populated zones of a node to the zonelist.
  1869. */
  1870. static int build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist,
  1871. int nr_zones, enum zone_type zone_type)
  1872. {
  1873. struct zone *zone;
  1874. BUG_ON(zone_type >= MAX_NR_ZONES);
  1875. zone_type++;
  1876. do {
  1877. zone_type--;
  1878. zone = pgdat->node_zones + zone_type;
  1879. if (populated_zone(zone)) {
  1880. zoneref_set_zone(zone,
  1881. &zonelist->_zonerefs[nr_zones++]);
  1882. check_highest_zone(zone_type);
  1883. }
  1884. } while (zone_type);
  1885. return nr_zones;
  1886. }
  1887. /*
  1888. * zonelist_order:
  1889. * 0 = automatic detection of better ordering.
  1890. * 1 = order by ([node] distance, -zonetype)
  1891. * 2 = order by (-zonetype, [node] distance)
  1892. *
  1893. * If not NUMA, ZONELIST_ORDER_ZONE and ZONELIST_ORDER_NODE will create
  1894. * the same zonelist. So only NUMA can configure this param.
  1895. */
  1896. #define ZONELIST_ORDER_DEFAULT 0
  1897. #define ZONELIST_ORDER_NODE 1
  1898. #define ZONELIST_ORDER_ZONE 2
  1899. /* zonelist order in the kernel.
  1900. * set_zonelist_order() will set this to NODE or ZONE.
  1901. */
  1902. static int current_zonelist_order = ZONELIST_ORDER_DEFAULT;
  1903. static char zonelist_order_name[3][8] = {"Default", "Node", "Zone"};
  1904. #ifdef CONFIG_NUMA
  1905. /* The value user specified ....changed by config */
  1906. static int user_zonelist_order = ZONELIST_ORDER_DEFAULT;
  1907. /* string for sysctl */
  1908. #define NUMA_ZONELIST_ORDER_LEN 16
  1909. char numa_zonelist_order[16] = "default";
  1910. /*
  1911. * interface for configure zonelist ordering.
  1912. * command line option "numa_zonelist_order"
  1913. * = "[dD]efault - default, automatic configuration.
  1914. * = "[nN]ode - order by node locality, then by zone within node
  1915. * = "[zZ]one - order by zone, then by locality within zone
  1916. */
  1917. static int __parse_numa_zonelist_order(char *s)
  1918. {
  1919. if (*s == 'd' || *s == 'D') {
  1920. user_zonelist_order = ZONELIST_ORDER_DEFAULT;
  1921. } else if (*s == 'n' || *s == 'N') {
  1922. user_zonelist_order = ZONELIST_ORDER_NODE;
  1923. } else if (*s == 'z' || *s == 'Z') {
  1924. user_zonelist_order = ZONELIST_ORDER_ZONE;
  1925. } else {
  1926. printk(KERN_WARNING
  1927. "Ignoring invalid numa_zonelist_order value: "
  1928. "%s\n", s);
  1929. return -EINVAL;
  1930. }
  1931. return 0;
  1932. }
  1933. static __init int setup_numa_zonelist_order(char *s)
  1934. {
  1935. if (s)
  1936. return __parse_numa_zonelist_order(s);
  1937. return 0;
  1938. }
  1939. early_param("numa_zonelist_order", setup_numa_zonelist_order);
  1940. /*
  1941. * sysctl handler for numa_zonelist_order
  1942. */
  1943. int numa_zonelist_order_handler(ctl_table *table, int write,
  1944. struct file *file, void __user *buffer, size_t *length,
  1945. loff_t *ppos)
  1946. {
  1947. char saved_string[NUMA_ZONELIST_ORDER_LEN];
  1948. int ret;
  1949. if (write)
  1950. strncpy(saved_string, (char*)table->data,
  1951. NUMA_ZONELIST_ORDER_LEN);
  1952. ret = proc_dostring(table, write, file, buffer, length, ppos);
  1953. if (ret)
  1954. return ret;
  1955. if (write) {
  1956. int oldval = user_zonelist_order;
  1957. if (__parse_numa_zonelist_order((char*)table->data)) {
  1958. /*
  1959. * bogus value. restore saved string
  1960. */
  1961. strncpy((char*)table->data, saved_string,
  1962. NUMA_ZONELIST_ORDER_LEN);
  1963. user_zonelist_order = oldval;
  1964. } else if (oldval != user_zonelist_order)
  1965. build_all_zonelists();
  1966. }
  1967. return 0;
  1968. }
  1969. #define MAX_NODE_LOAD (nr_online_nodes)
  1970. static int node_load[MAX_NUMNODES];
  1971. /**
  1972. * find_next_best_node - find the next node that should appear in a given node's fallback list
  1973. * @node: node whose fallback list we're appending
  1974. * @used_node_mask: nodemask_t of already used nodes
  1975. *
  1976. * We use a number of factors to determine which is the next node that should
  1977. * appear on a given node's fallback list. The node should not have appeared
  1978. * already in @node's fallback list, and it should be the next closest node
  1979. * according to the distance array (which contains arbitrary distance values
  1980. * from each node to each node in the system), and should also prefer nodes
  1981. * with no CPUs, since presumably they'll have very little allocation pressure
  1982. * on them otherwise.
  1983. * It returns -1 if no node is found.
  1984. */
  1985. static int find_next_best_node(int node, nodemask_t *used_node_mask)
  1986. {
  1987. int n, val;
  1988. int min_val = INT_MAX;
  1989. int best_node = -1;
  1990. const struct cpumask *tmp = cpumask_of_node(0);
  1991. /* Use the local node if we haven't already */
  1992. if (!node_isset(node, *used_node_mask)) {
  1993. node_set(node, *used_node_mask);
  1994. return node;
  1995. }
  1996. for_each_node_state(n, N_HIGH_MEMORY) {
  1997. /* Don't want a node to appear more than once */
  1998. if (node_isset(n, *used_node_mask))
  1999. continue;
  2000. /* Use the distance array to find the distance */
  2001. val = node_distance(node, n);
  2002. /* Penalize nodes under us ("prefer the next node") */
  2003. val += (n < node);
  2004. /* Give preference to headless and unused nodes */
  2005. tmp = cpumask_of_node(n);
  2006. if (!cpumask_empty(tmp))
  2007. val += PENALTY_FOR_NODE_WITH_CPUS;
  2008. /* Slight preference for less loaded node */
  2009. val *= (MAX_NODE_LOAD*MAX_NUMNODES);
  2010. val += node_load[n];
  2011. if (val < min_val) {
  2012. min_val = val;
  2013. best_node = n;
  2014. }
  2015. }
  2016. if (best_node >= 0)
  2017. node_set(best_node, *used_node_mask);
  2018. return best_node;
  2019. }
  2020. /*
  2021. * Build zonelists ordered by node and zones within node.
  2022. * This results in maximum locality--normal zone overflows into local
  2023. * DMA zone, if any--but risks exhausting DMA zone.
  2024. */
  2025. static void build_zonelists_in_node_order(pg_data_t *pgdat, int node)
  2026. {
  2027. int j;
  2028. struct zonelist *zonelist;
  2029. zonelist = &pgdat->node_zonelists[0];
  2030. for (j = 0; zonelist->_zonerefs[j].zone != NULL; j++)
  2031. ;
  2032. j = build_zonelists_node(NODE_DATA(node), zonelist, j,
  2033. MAX_NR_ZONES - 1);
  2034. zonelist->_zonerefs[j].zone = NULL;
  2035. zonelist->_zonerefs[j].zone_idx = 0;
  2036. }
  2037. /*
  2038. * Build gfp_thisnode zonelists
  2039. */
  2040. static void build_thisnode_zonelists(pg_data_t *pgdat)
  2041. {
  2042. int j;
  2043. struct zonelist *zonelist;
  2044. zonelist = &pgdat->node_zonelists[1];
  2045. j = build_zonelists_node(pgdat, zonelist, 0, MAX_NR_ZONES - 1);
  2046. zonelist->_zonerefs[j].zone = NULL;
  2047. zonelist->_zonerefs[j].zone_idx = 0;
  2048. }
  2049. /*
  2050. * Build zonelists ordered by zone and nodes within zones.
  2051. * This results in conserving DMA zone[s] until all Normal memory is
  2052. * exhausted, but results in overflowing to remote node while memory
  2053. * may still exist in local DMA zone.
  2054. */
  2055. static int node_order[MAX_NUMNODES];
  2056. static void build_zonelists_in_zone_order(pg_data_t *pgdat, int nr_nodes)
  2057. {
  2058. int pos, j, node;
  2059. int zone_type; /* needs to be signed */
  2060. struct zone *z;
  2061. struct zonelist *zonelist;
  2062. zonelist = &pgdat->node_zonelists[0];
  2063. pos = 0;
  2064. for (zone_type = MAX_NR_ZONES - 1; zone_type >= 0; zone_type--) {
  2065. for (j = 0; j < nr_nodes; j++) {
  2066. node = node_order[j];
  2067. z = &NODE_DATA(node)->node_zones[zone_type];
  2068. if (populated_zone(z)) {
  2069. zoneref_set_zone(z,
  2070. &zonelist->_zonerefs[pos++]);
  2071. check_highest_zone(zone_type);
  2072. }
  2073. }
  2074. }
  2075. zonelist->_zonerefs[pos].zone = NULL;
  2076. zonelist->_zonerefs[pos].zone_idx = 0;
  2077. }
  2078. static int default_zonelist_order(void)
  2079. {
  2080. int nid, zone_type;
  2081. unsigned long low_kmem_size,total_size;
  2082. struct zone *z;
  2083. int average_size;
  2084. /*
  2085. * ZONE_DMA and ZONE_DMA32 can be very small area in the sytem.
  2086. * If they are really small and used heavily, the system can fall
  2087. * into OOM very easily.
  2088. * This function detect ZONE_DMA/DMA32 size and confgigures zone order.
  2089. */
  2090. /* Is there ZONE_NORMAL ? (ex. ppc has only DMA zone..) */
  2091. low_kmem_size = 0;
  2092. total_size = 0;
  2093. for_each_online_node(nid) {
  2094. for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
  2095. z = &NODE_DATA(nid)->node_zones[zone_type];
  2096. if (populated_zone(z)) {
  2097. if (zone_type < ZONE_NORMAL)
  2098. low_kmem_size += z->present_pages;
  2099. total_size += z->present_pages;
  2100. }
  2101. }
  2102. }
  2103. if (!low_kmem_size || /* there are no DMA area. */
  2104. low_kmem_size > total_size/2) /* DMA/DMA32 is big. */
  2105. return ZONELIST_ORDER_NODE;
  2106. /*
  2107. * look into each node's config.
  2108. * If there is a node whose DMA/DMA32 memory is very big area on
  2109. * local memory, NODE_ORDER may be suitable.
  2110. */
  2111. average_size = total_size /
  2112. (nodes_weight(node_states[N_HIGH_MEMORY]) + 1);
  2113. for_each_online_node(nid) {
  2114. low_kmem_size = 0;
  2115. total_size = 0;
  2116. for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
  2117. z = &NODE_DATA(nid)->node_zones[zone_type];
  2118. if (populated_zone(z)) {
  2119. if (zone_type < ZONE_NORMAL)
  2120. low_kmem_size += z->present_pages;
  2121. total_size += z->present_pages;
  2122. }
  2123. }
  2124. if (low_kmem_size &&
  2125. total_size > average_size && /* ignore small node */
  2126. low_kmem_size > total_size * 70/100)
  2127. return ZONELIST_ORDER_NODE;
  2128. }
  2129. return ZONELIST_ORDER_ZONE;
  2130. }
  2131. static void set_zonelist_order(void)
  2132. {
  2133. if (user_zonelist_order == ZONELIST_ORDER_DEFAULT)
  2134. current_zonelist_order = default_zonelist_order();
  2135. else
  2136. current_zonelist_order = user_zonelist_order;
  2137. }
  2138. static void build_zonelists(pg_data_t *pgdat)
  2139. {
  2140. int j, node, load;
  2141. enum zone_type i;
  2142. nodemask_t used_mask;
  2143. int local_node, prev_node;
  2144. struct zonelist *zonelist;
  2145. int order = current_zonelist_order;
  2146. /* initialize zonelists */
  2147. for (i = 0; i < MAX_ZONELISTS; i++) {
  2148. zonelist = pgdat->node_zonelists + i;
  2149. zonelist->_zonerefs[0].zone = NULL;
  2150. zonelist->_zonerefs[0].zone_idx = 0;
  2151. }
  2152. /* NUMA-aware ordering of nodes */
  2153. local_node = pgdat->node_id;
  2154. load = nr_online_nodes;
  2155. prev_node = local_node;
  2156. nodes_clear(used_mask);
  2157. memset(node_load, 0, sizeof(node_load));
  2158. memset(node_order, 0, sizeof(node_order));
  2159. j = 0;
  2160. while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
  2161. int distance = node_distance(local_node, node);
  2162. /*
  2163. * If another node is sufficiently far away then it is better
  2164. * to reclaim pages in a zone before going off node.
  2165. */
  2166. if (distance > RECLAIM_DISTANCE)
  2167. zone_reclaim_mode = 1;
  2168. /*
  2169. * We don't want to pressure a particular node.
  2170. * So adding penalty to the first node in same
  2171. * distance group to make it round-robin.
  2172. */
  2173. if (distance != node_distance(local_node, prev_node))
  2174. node_load[node] = load;
  2175. prev_node = node;
  2176. load--;
  2177. if (order == ZONELIST_ORDER_NODE)
  2178. build_zonelists_in_node_order(pgdat, node);
  2179. else
  2180. node_order[j++] = node; /* remember order */
  2181. }
  2182. if (order == ZONELIST_ORDER_ZONE) {
  2183. /* calculate node order -- i.e., DMA last! */
  2184. build_zonelists_in_zone_order(pgdat, j);
  2185. }
  2186. build_thisnode_zonelists(pgdat);
  2187. }
  2188. /* Construct the zonelist performance cache - see further mmzone.h */
  2189. static void build_zonelist_cache(pg_data_t *pgdat)
  2190. {
  2191. struct zonelist *zonelist;
  2192. struct zonelist_cache *zlc;
  2193. struct zoneref *z;
  2194. zonelist = &pgdat->node_zonelists[0];
  2195. zonelist->zlcache_ptr = zlc = &zonelist->zlcache;
  2196. bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
  2197. for (z = zonelist->_zonerefs; z->zone; z++)
  2198. zlc->z_to_n[z - zonelist->_zonerefs] = zonelist_node_idx(z);
  2199. }
  2200. #else /* CONFIG_NUMA */
  2201. static void set_zonelist_order(void)
  2202. {
  2203. current_zonelist_order = ZONELIST_ORDER_ZONE;
  2204. }
  2205. static void build_zonelists(pg_data_t *pgdat)
  2206. {
  2207. int node, local_node;
  2208. enum zone_type j;
  2209. struct zonelist *zonelist;
  2210. local_node = pgdat->node_id;
  2211. zonelist = &pgdat->node_zonelists[0];
  2212. j = build_zonelists_node(pgdat, zonelist, 0, MAX_NR_ZONES - 1);
  2213. /*
  2214. * Now we build the zonelist so that it contains the zones
  2215. * of all the other nodes.
  2216. * We don't want to pressure a particular node, so when
  2217. * building the zones for node N, we make sure that the
  2218. * zones coming right after the local ones are those from
  2219. * node N+1 (modulo N)
  2220. */
  2221. for (node = local_node + 1; node < MAX_NUMNODES; node++) {
  2222. if (!node_online(node))
  2223. continue;
  2224. j = build_zonelists_node(NODE_DATA(node), zonelist, j,
  2225. MAX_NR_ZONES - 1);
  2226. }
  2227. for (node = 0; node < local_node; node++) {
  2228. if (!node_online(node))
  2229. continue;
  2230. j = build_zonelists_node(NODE_DATA(node), zonelist, j,
  2231. MAX_NR_ZONES - 1);
  2232. }
  2233. zonelist->_zonerefs[j].zone = NULL;
  2234. zonelist->_zonerefs[j].zone_idx = 0;
  2235. }
  2236. /* non-NUMA variant of zonelist performance cache - just NULL zlcache_ptr */
  2237. static void build_zonelist_cache(pg_data_t *pgdat)
  2238. {
  2239. pgdat->node_zonelists[0].zlcache_ptr = NULL;
  2240. }
  2241. #endif /* CONFIG_NUMA */
  2242. /* return values int ....just for stop_machine() */
  2243. static int __build_all_zonelists(void *dummy)
  2244. {
  2245. int nid;
  2246. for_each_online_node(nid) {
  2247. pg_data_t *pgdat = NODE_DATA(nid);
  2248. build_zonelists(pgdat);
  2249. build_zonelist_cache(pgdat);
  2250. }
  2251. return 0;
  2252. }
  2253. void build_all_zonelists(void)
  2254. {
  2255. set_zonelist_order();
  2256. if (system_state == SYSTEM_BOOTING) {
  2257. __build_all_zonelists(NULL);
  2258. mminit_verify_zonelist();
  2259. cpuset_init_current_mems_allowed();
  2260. } else {
  2261. /* we have to stop all cpus to guarantee there is no user
  2262. of zonelist */
  2263. stop_machine(__build_all_zonelists, NULL, NULL);
  2264. /* cpuset refresh routine should be here */
  2265. }
  2266. vm_total_pages = nr_free_pagecache_pages();
  2267. /*
  2268. * Disable grouping by mobility if the number of pages in the
  2269. * system is too low to allow the mechanism to work. It would be
  2270. * more accurate, but expensive to check per-zone. This check is
  2271. * made on memory-hotadd so a system can start with mobility
  2272. * disabled and enable it later
  2273. */
  2274. if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
  2275. page_group_by_mobility_disabled = 1;
  2276. else
  2277. page_group_by_mobility_disabled = 0;
  2278. printk("Built %i zonelists in %s order, mobility grouping %s. "
  2279. "Total pages: %ld\n",
  2280. nr_online_nodes,
  2281. zonelist_order_name[current_zonelist_order],
  2282. page_group_by_mobility_disabled ? "off" : "on",
  2283. vm_total_pages);
  2284. #ifdef CONFIG_NUMA
  2285. printk("Policy zone: %s\n", zone_names[policy_zone]);
  2286. #endif
  2287. }
  2288. /*
  2289. * Helper functions to size the waitqueue hash table.
  2290. * Essentially these want to choose hash table sizes sufficiently
  2291. * large so that collisions trying to wait on pages are rare.
  2292. * But in fact, the number of active page waitqueues on typical
  2293. * systems is ridiculously low, less than 200. So this is even
  2294. * conservative, even though it seems large.
  2295. *
  2296. * The constant PAGES_PER_WAITQUEUE specifies the ratio of pages to
  2297. * waitqueues, i.e. the size of the waitq table given the number of pages.
  2298. */
  2299. #define PAGES_PER_WAITQUEUE 256
  2300. #ifndef CONFIG_MEMORY_HOTPLUG
  2301. static inline unsigned long wait_table_hash_nr_entries(unsigned long pages)
  2302. {
  2303. unsigned long size = 1;
  2304. pages /= PAGES_PER_WAITQUEUE;
  2305. while (size < pages)
  2306. size <<= 1;
  2307. /*
  2308. * Once we have dozens or even hundreds of threads sleeping
  2309. * on IO we've got bigger problems than wait queue collision.
  2310. * Limit the size of the wait table to a reasonable size.
  2311. */
  2312. size = min(size, 4096UL);
  2313. return max(size, 4UL);
  2314. }
  2315. #else
  2316. /*
  2317. * A zone's size might be changed by hot-add, so it is not possible to determine
  2318. * a suitable size for its wait_table. So we use the maximum size now.
  2319. *
  2320. * The max wait table size = 4096 x sizeof(wait_queue_head_t). ie:
  2321. *
  2322. * i386 (preemption config) : 4096 x 16 = 64Kbyte.
  2323. * ia64, x86-64 (no preemption): 4096 x 20 = 80Kbyte.
  2324. * ia64, x86-64 (preemption) : 4096 x 24 = 96Kbyte.
  2325. *
  2326. * The maximum entries are prepared when a zone's memory is (512K + 256) pages
  2327. * or more by the traditional way. (See above). It equals:
  2328. *
  2329. * i386, x86-64, powerpc(4K page size) : = ( 2G + 1M)byte.
  2330. * ia64(16K page size) : = ( 8G + 4M)byte.
  2331. * powerpc (64K page size) : = (32G +16M)byte.
  2332. */
  2333. static inline unsigned long wait_table_hash_nr_entries(unsigned long pages)
  2334. {
  2335. return 4096UL;
  2336. }
  2337. #endif
  2338. /*
  2339. * This is an integer logarithm so that shifts can be used later
  2340. * to extract the more random high bits from the multiplicative
  2341. * hash function before the remainder is taken.
  2342. */
  2343. static inline unsigned long wait_table_bits(unsigned long size)
  2344. {
  2345. return ffz(~size);
  2346. }
  2347. #define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1))
  2348. /*
  2349. * Mark a number of pageblocks as MIGRATE_RESERVE. The number
  2350. * of blocks reserved is based on min_wmark_pages(zone). The memory within
  2351. * the reserve will tend to store contiguous free pages. Setting min_free_kbytes
  2352. * higher will lead to a bigger reserve which will get freed as contiguous
  2353. * blocks as reclaim kicks in
  2354. */
  2355. static void setup_zone_migrate_reserve(struct zone *zone)
  2356. {
  2357. unsigned long start_pfn, pfn, end_pfn;
  2358. struct page *page;
  2359. unsigned long reserve, block_migratetype;
  2360. /* Get the start pfn, end pfn and the number of blocks to reserve */
  2361. start_pfn = zone->zone_start_pfn;
  2362. end_pfn = start_pfn + zone->spanned_pages;
  2363. reserve = roundup(min_wmark_pages(zone), pageblock_nr_pages) >>
  2364. pageblock_order;
  2365. for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
  2366. if (!pfn_valid(pfn))
  2367. continue;
  2368. page = pfn_to_page(pfn);
  2369. /* Watch out for overlapping nodes */
  2370. if (page_to_nid(page) != zone_to_nid(zone))
  2371. continue;
  2372. /* Blocks with reserved pages will never free, skip them. */
  2373. if (PageReserved(page))
  2374. continue;
  2375. block_migratetype = get_pageblock_migratetype(page);
  2376. /* If this block is reserved, account for it */
  2377. if (reserve > 0 && block_migratetype == MIGRATE_RESERVE) {
  2378. reserve--;
  2379. continue;
  2380. }
  2381. /* Suitable for reserving if this block is movable */
  2382. if (reserve > 0 && block_migratetype == MIGRATE_MOVABLE) {
  2383. set_pageblock_migratetype(page, MIGRATE_RESERVE);
  2384. move_freepages_block(zone, page, MIGRATE_RESERVE);
  2385. reserve--;
  2386. continue;
  2387. }
  2388. /*
  2389. * If the reserve is met and this is a previous reserved block,
  2390. * take it back
  2391. */
  2392. if (block_migratetype == MIGRATE_RESERVE) {
  2393. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  2394. move_freepages_block(zone, page, MIGRATE_MOVABLE);
  2395. }
  2396. }
  2397. }
  2398. /*
  2399. * Initially all pages are reserved - free ones are freed
  2400. * up by free_all_bootmem() once the early boot process is
  2401. * done. Non-atomic initialization, single-pass.
  2402. */
  2403. void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
  2404. unsigned long start_pfn, enum memmap_context context)
  2405. {
  2406. struct page *page;
  2407. unsigned long end_pfn = start_pfn + size;
  2408. unsigned long pfn;
  2409. struct zone *z;
  2410. if (highest_memmap_pfn < end_pfn - 1)
  2411. highest_memmap_pfn = end_pfn - 1;
  2412. z = &NODE_DATA(nid)->node_zones[zone];
  2413. for (pfn = start_pfn; pfn < end_pfn; pfn++) {
  2414. /*
  2415. * There can be holes in boot-time mem_map[]s
  2416. * handed to this function. They do not
  2417. * exist on hotplugged memory.
  2418. */
  2419. if (context == MEMMAP_EARLY) {
  2420. if (!early_pfn_valid(pfn))
  2421. continue;
  2422. if (!early_pfn_in_nid(pfn, nid))
  2423. continue;
  2424. }
  2425. page = pfn_to_page(pfn);
  2426. set_page_links(page, zone, nid, pfn);
  2427. mminit_verify_page_links(page, zone, nid, pfn);
  2428. init_page_count(page);
  2429. reset_page_mapcount(page);
  2430. SetPageReserved(page);
  2431. /*
  2432. * Mark the block movable so that blocks are reserved for
  2433. * movable at startup. This will force kernel allocations
  2434. * to reserve their blocks rather than leaking throughout
  2435. * the address space during boot when many long-lived
  2436. * kernel allocations are made. Later some blocks near
  2437. * the start are marked MIGRATE_RESERVE by
  2438. * setup_zone_migrate_reserve()
  2439. *
  2440. * bitmap is created for zone's valid pfn range. but memmap
  2441. * can be created for invalid pages (for alignment)
  2442. * check here not to call set_pageblock_migratetype() against
  2443. * pfn out of zone.
  2444. */
  2445. if ((z->zone_start_pfn <= pfn)
  2446. && (pfn < z->zone_start_pfn + z->spanned_pages)
  2447. && !(pfn & (pageblock_nr_pages - 1)))
  2448. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  2449. INIT_LIST_HEAD(&page->lru);
  2450. #ifdef WANT_PAGE_VIRTUAL
  2451. /* The shift won't overflow because ZONE_NORMAL is below 4G. */
  2452. if (!is_highmem_idx(zone))
  2453. set_page_address(page, __va(pfn << PAGE_SHIFT));
  2454. #endif
  2455. }
  2456. }
  2457. static void __meminit zone_init_free_lists(struct zone *zone)
  2458. {
  2459. int order, t;
  2460. for_each_migratetype_order(order, t) {
  2461. INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
  2462. zone->free_area[order].nr_free = 0;
  2463. }
  2464. }
  2465. #ifndef __HAVE_ARCH_MEMMAP_INIT
  2466. #define memmap_init(size, nid, zone, start_pfn) \
  2467. memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY)
  2468. #endif
  2469. static int zone_batchsize(struct zone *zone)
  2470. {
  2471. #ifdef CONFIG_MMU
  2472. int batch;
  2473. /*
  2474. * The per-cpu-pages pools are set to around 1000th of the
  2475. * size of the zone. But no more than 1/2 of a meg.
  2476. *
  2477. * OK, so we don't know how big the cache is. So guess.
  2478. */
  2479. batch = zone->present_pages / 1024;
  2480. if (batch * PAGE_SIZE > 512 * 1024)
  2481. batch = (512 * 1024) / PAGE_SIZE;
  2482. batch /= 4; /* We effectively *= 4 below */
  2483. if (batch < 1)
  2484. batch = 1;
  2485. /*
  2486. * Clamp the batch to a 2^n - 1 value. Having a power
  2487. * of 2 value was found to be more likely to have
  2488. * suboptimal cache aliasing properties in some cases.
  2489. *
  2490. * For example if 2 tasks are alternately allocating
  2491. * batches of pages, one task can end up with a lot
  2492. * of pages of one half of the possible page colors
  2493. * and the other with pages of the other colors.
  2494. */
  2495. batch = rounddown_pow_of_two(batch + batch/2) - 1;
  2496. return batch;
  2497. #else
  2498. /* The deferral and batching of frees should be suppressed under NOMMU
  2499. * conditions.
  2500. *
  2501. * The problem is that NOMMU needs to be able to allocate large chunks
  2502. * of contiguous memory as there's no hardware page translation to
  2503. * assemble apparent contiguous memory from discontiguous pages.
  2504. *
  2505. * Queueing large contiguous runs of pages for batching, however,
  2506. * causes the pages to actually be freed in smaller chunks. As there
  2507. * can be a significant delay between the individual batches being
  2508. * recycled, this leads to the once large chunks of space being
  2509. * fragmented and becoming unavailable for high-order allocations.
  2510. */
  2511. return 0;
  2512. #endif
  2513. }
  2514. static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
  2515. {
  2516. struct per_cpu_pages *pcp;
  2517. memset(p, 0, sizeof(*p));
  2518. pcp = &p->pcp;
  2519. pcp->count = 0;
  2520. pcp->high = 6 * batch;
  2521. pcp->batch = max(1UL, 1 * batch);
  2522. INIT_LIST_HEAD(&pcp->list);
  2523. }
  2524. /*
  2525. * setup_pagelist_highmark() sets the high water mark for hot per_cpu_pagelist
  2526. * to the value high for the pageset p.
  2527. */
  2528. static void setup_pagelist_highmark(struct per_cpu_pageset *p,
  2529. unsigned long high)
  2530. {
  2531. struct per_cpu_pages *pcp;
  2532. pcp = &p->pcp;
  2533. pcp->high = high;
  2534. pcp->batch = max(1UL, high/4);
  2535. if ((high/4) > (PAGE_SHIFT * 8))
  2536. pcp->batch = PAGE_SHIFT * 8;
  2537. }
  2538. #ifdef CONFIG_NUMA
  2539. /*
  2540. * Boot pageset table. One per cpu which is going to be used for all
  2541. * zones and all nodes. The parameters will be set in such a way
  2542. * that an item put on a list will immediately be handed over to
  2543. * the buddy list. This is safe since pageset manipulation is done
  2544. * with interrupts disabled.
  2545. *
  2546. * Some NUMA counter updates may also be caught by the boot pagesets.
  2547. *
  2548. * The boot_pagesets must be kept even after bootup is complete for
  2549. * unused processors and/or zones. They do play a role for bootstrapping
  2550. * hotplugged processors.
  2551. *
  2552. * zoneinfo_show() and maybe other functions do
  2553. * not check if the processor is online before following the pageset pointer.
  2554. * Other parts of the kernel may not check if the zone is available.
  2555. */
  2556. static struct per_cpu_pageset boot_pageset[NR_CPUS];
  2557. /*
  2558. * Dynamically allocate memory for the
  2559. * per cpu pageset array in struct zone.
  2560. */
  2561. static int __cpuinit process_zones(int cpu)
  2562. {
  2563. struct zone *zone, *dzone;
  2564. int node = cpu_to_node(cpu);
  2565. node_set_state(node, N_CPU); /* this node has a cpu */
  2566. for_each_populated_zone(zone) {
  2567. zone_pcp(zone, cpu) = kmalloc_node(sizeof(struct per_cpu_pageset),
  2568. GFP_KERNEL, node);
  2569. if (!zone_pcp(zone, cpu))
  2570. goto bad;
  2571. setup_pageset(zone_pcp(zone, cpu), zone_batchsize(zone));
  2572. if (percpu_pagelist_fraction)
  2573. setup_pagelist_highmark(zone_pcp(zone, cpu),
  2574. (zone->present_pages / percpu_pagelist_fraction));
  2575. }
  2576. return 0;
  2577. bad:
  2578. for_each_zone(dzone) {
  2579. if (!populated_zone(dzone))
  2580. continue;
  2581. if (dzone == zone)
  2582. break;
  2583. kfree(zone_pcp(dzone, cpu));
  2584. zone_pcp(dzone, cpu) = NULL;
  2585. }
  2586. return -ENOMEM;
  2587. }
  2588. static inline void free_zone_pagesets(int cpu)
  2589. {
  2590. struct zone *zone;
  2591. for_each_zone(zone) {
  2592. struct per_cpu_pageset *pset = zone_pcp(zone, cpu);
  2593. /* Free per_cpu_pageset if it is slab allocated */
  2594. if (pset != &boot_pageset[cpu])
  2595. kfree(pset);
  2596. zone_pcp(zone, cpu) = NULL;
  2597. }
  2598. }
  2599. static int __cpuinit pageset_cpuup_callback(struct notifier_block *nfb,
  2600. unsigned long action,
  2601. void *hcpu)
  2602. {
  2603. int cpu = (long)hcpu;
  2604. int ret = NOTIFY_OK;
  2605. switch (action) {
  2606. case CPU_UP_PREPARE:
  2607. case CPU_UP_PREPARE_FROZEN:
  2608. if (process_zones(cpu))
  2609. ret = NOTIFY_BAD;
  2610. break;
  2611. case CPU_UP_CANCELED:
  2612. case CPU_UP_CANCELED_FROZEN:
  2613. case CPU_DEAD:
  2614. case CPU_DEAD_FROZEN:
  2615. free_zone_pagesets(cpu);
  2616. break;
  2617. default:
  2618. break;
  2619. }
  2620. return ret;
  2621. }
  2622. static struct notifier_block __cpuinitdata pageset_notifier =
  2623. { &pageset_cpuup_callback, NULL, 0 };
  2624. void __init setup_per_cpu_pageset(void)
  2625. {
  2626. int err;
  2627. /* Initialize per_cpu_pageset for cpu 0.
  2628. * A cpuup callback will do this for every cpu
  2629. * as it comes online
  2630. */
  2631. err = process_zones(smp_processor_id());
  2632. BUG_ON(err);
  2633. register_cpu_notifier(&pageset_notifier);
  2634. }
  2635. #endif
  2636. static noinline __init_refok
  2637. int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages)
  2638. {
  2639. int i;
  2640. struct pglist_data *pgdat = zone->zone_pgdat;
  2641. size_t alloc_size;
  2642. /*
  2643. * The per-page waitqueue mechanism uses hashed waitqueues
  2644. * per zone.
  2645. */
  2646. zone->wait_table_hash_nr_entries =
  2647. wait_table_hash_nr_entries(zone_size_pages);
  2648. zone->wait_table_bits =
  2649. wait_table_bits(zone->wait_table_hash_nr_entries);
  2650. alloc_size = zone->wait_table_hash_nr_entries
  2651. * sizeof(wait_queue_head_t);
  2652. if (!slab_is_available()) {
  2653. zone->wait_table = (wait_queue_head_t *)
  2654. alloc_bootmem_node(pgdat, alloc_size);
  2655. } else {
  2656. /*
  2657. * This case means that a zone whose size was 0 gets new memory
  2658. * via memory hot-add.
  2659. * But it may be the case that a new node was hot-added. In
  2660. * this case vmalloc() will not be able to use this new node's
  2661. * memory - this wait_table must be initialized to use this new
  2662. * node itself as well.
  2663. * To use this new node's memory, further consideration will be
  2664. * necessary.
  2665. */
  2666. zone->wait_table = vmalloc(alloc_size);
  2667. }
  2668. if (!zone->wait_table)
  2669. return -ENOMEM;
  2670. for(i = 0; i < zone->wait_table_hash_nr_entries; ++i)
  2671. init_waitqueue_head(zone->wait_table + i);
  2672. return 0;
  2673. }
  2674. static __meminit void zone_pcp_init(struct zone *zone)
  2675. {
  2676. int cpu;
  2677. unsigned long batch = zone_batchsize(zone);
  2678. for (cpu = 0; cpu < NR_CPUS; cpu++) {
  2679. #ifdef CONFIG_NUMA
  2680. /* Early boot. Slab allocator not functional yet */
  2681. zone_pcp(zone, cpu) = &boot_pageset[cpu];
  2682. setup_pageset(&boot_pageset[cpu],0);
  2683. #else
  2684. setup_pageset(zone_pcp(zone,cpu), batch);
  2685. #endif
  2686. }
  2687. if (zone->present_pages)
  2688. printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%lu\n",
  2689. zone->name, zone->present_pages, batch);
  2690. }
  2691. __meminit int init_currently_empty_zone(struct zone *zone,
  2692. unsigned long zone_start_pfn,
  2693. unsigned long size,
  2694. enum memmap_context context)
  2695. {
  2696. struct pglist_data *pgdat = zone->zone_pgdat;
  2697. int ret;
  2698. ret = zone_wait_table_init(zone, size);
  2699. if (ret)
  2700. return ret;
  2701. pgdat->nr_zones = zone_idx(zone) + 1;
  2702. zone->zone_start_pfn = zone_start_pfn;
  2703. mminit_dprintk(MMINIT_TRACE, "memmap_init",
  2704. "Initialising map node %d zone %lu pfns %lu -> %lu\n",
  2705. pgdat->node_id,
  2706. (unsigned long)zone_idx(zone),
  2707. zone_start_pfn, (zone_start_pfn + size));
  2708. zone_init_free_lists(zone);
  2709. return 0;
  2710. }
  2711. #ifdef CONFIG_ARCH_POPULATES_NODE_MAP
  2712. /*
  2713. * Basic iterator support. Return the first range of PFNs for a node
  2714. * Note: nid == MAX_NUMNODES returns first region regardless of node
  2715. */
  2716. static int __meminit first_active_region_index_in_nid(int nid)
  2717. {
  2718. int i;
  2719. for (i = 0; i < nr_nodemap_entries; i++)
  2720. if (nid == MAX_NUMNODES || early_node_map[i].nid == nid)
  2721. return i;
  2722. return -1;
  2723. }
  2724. /*
  2725. * Basic iterator support. Return the next active range of PFNs for a node
  2726. * Note: nid == MAX_NUMNODES returns next region regardless of node
  2727. */
  2728. static int __meminit next_active_region_index_in_nid(int index, int nid)
  2729. {
  2730. for (index = index + 1; index < nr_nodemap_entries; index++)
  2731. if (nid == MAX_NUMNODES || early_node_map[index].nid == nid)
  2732. return index;
  2733. return -1;
  2734. }
  2735. #ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
  2736. /*
  2737. * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
  2738. * Architectures may implement their own version but if add_active_range()
  2739. * was used and there are no special requirements, this is a convenient
  2740. * alternative
  2741. */
  2742. int __meminit __early_pfn_to_nid(unsigned long pfn)
  2743. {
  2744. int i;
  2745. for (i = 0; i < nr_nodemap_entries; i++) {
  2746. unsigned long start_pfn = early_node_map[i].start_pfn;
  2747. unsigned long end_pfn = early_node_map[i].end_pfn;
  2748. if (start_pfn <= pfn && pfn < end_pfn)
  2749. return early_node_map[i].nid;
  2750. }
  2751. /* This is a memory hole */
  2752. return -1;
  2753. }
  2754. #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
  2755. int __meminit early_pfn_to_nid(unsigned long pfn)
  2756. {
  2757. int nid;
  2758. nid = __early_pfn_to_nid(pfn);
  2759. if (nid >= 0)
  2760. return nid;
  2761. /* just returns 0 */
  2762. return 0;
  2763. }
  2764. #ifdef CONFIG_NODES_SPAN_OTHER_NODES
  2765. bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
  2766. {
  2767. int nid;
  2768. nid = __early_pfn_to_nid(pfn);
  2769. if (nid >= 0 && nid != node)
  2770. return false;
  2771. return true;
  2772. }
  2773. #endif
  2774. /* Basic iterator support to walk early_node_map[] */
  2775. #define for_each_active_range_index_in_nid(i, nid) \
  2776. for (i = first_active_region_index_in_nid(nid); i != -1; \
  2777. i = next_active_region_index_in_nid(i, nid))
  2778. /**
  2779. * free_bootmem_with_active_regions - Call free_bootmem_node for each active range
  2780. * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed.
  2781. * @max_low_pfn: The highest PFN that will be passed to free_bootmem_node
  2782. *
  2783. * If an architecture guarantees that all ranges registered with
  2784. * add_active_ranges() contain no holes and may be freed, this
  2785. * this function may be used instead of calling free_bootmem() manually.
  2786. */
  2787. void __init free_bootmem_with_active_regions(int nid,
  2788. unsigned long max_low_pfn)
  2789. {
  2790. int i;
  2791. for_each_active_range_index_in_nid(i, nid) {
  2792. unsigned long size_pages = 0;
  2793. unsigned long end_pfn = early_node_map[i].end_pfn;
  2794. if (early_node_map[i].start_pfn >= max_low_pfn)
  2795. continue;
  2796. if (end_pfn > max_low_pfn)
  2797. end_pfn = max_low_pfn;
  2798. size_pages = end_pfn - early_node_map[i].start_pfn;
  2799. free_bootmem_node(NODE_DATA(early_node_map[i].nid),
  2800. PFN_PHYS(early_node_map[i].start_pfn),
  2801. size_pages << PAGE_SHIFT);
  2802. }
  2803. }
  2804. void __init work_with_active_regions(int nid, work_fn_t work_fn, void *data)
  2805. {
  2806. int i;
  2807. int ret;
  2808. for_each_active_range_index_in_nid(i, nid) {
  2809. ret = work_fn(early_node_map[i].start_pfn,
  2810. early_node_map[i].end_pfn, data);
  2811. if (ret)
  2812. break;
  2813. }
  2814. }
  2815. /**
  2816. * sparse_memory_present_with_active_regions - Call memory_present for each active range
  2817. * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used.
  2818. *
  2819. * If an architecture guarantees that all ranges registered with
  2820. * add_active_ranges() contain no holes and may be freed, this
  2821. * function may be used instead of calling memory_present() manually.
  2822. */
  2823. void __init sparse_memory_present_with_active_regions(int nid)
  2824. {
  2825. int i;
  2826. for_each_active_range_index_in_nid(i, nid)
  2827. memory_present(early_node_map[i].nid,
  2828. early_node_map[i].start_pfn,
  2829. early_node_map[i].end_pfn);
  2830. }
  2831. /**
  2832. * get_pfn_range_for_nid - Return the start and end page frames for a node
  2833. * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
  2834. * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
  2835. * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
  2836. *
  2837. * It returns the start and end page frame of a node based on information
  2838. * provided by an arch calling add_active_range(). If called for a node
  2839. * with no available memory, a warning is printed and the start and end
  2840. * PFNs will be 0.
  2841. */
  2842. void __meminit get_pfn_range_for_nid(unsigned int nid,
  2843. unsigned long *start_pfn, unsigned long *end_pfn)
  2844. {
  2845. int i;
  2846. *start_pfn = -1UL;
  2847. *end_pfn = 0;
  2848. for_each_active_range_index_in_nid(i, nid) {
  2849. *start_pfn = min(*start_pfn, early_node_map[i].start_pfn);
  2850. *end_pfn = max(*end_pfn, early_node_map[i].end_pfn);
  2851. }
  2852. if (*start_pfn == -1UL)
  2853. *start_pfn = 0;
  2854. }
  2855. /*
  2856. * This finds a zone that can be used for ZONE_MOVABLE pages. The
  2857. * assumption is made that zones within a node are ordered in monotonic
  2858. * increasing memory addresses so that the "highest" populated zone is used
  2859. */
  2860. static void __init find_usable_zone_for_movable(void)
  2861. {
  2862. int zone_index;
  2863. for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
  2864. if (zone_index == ZONE_MOVABLE)
  2865. continue;
  2866. if (arch_zone_highest_possible_pfn[zone_index] >
  2867. arch_zone_lowest_possible_pfn[zone_index])
  2868. break;
  2869. }
  2870. VM_BUG_ON(zone_index == -1);
  2871. movable_zone = zone_index;
  2872. }
  2873. /*
  2874. * The zone ranges provided by the architecture do not include ZONE_MOVABLE
  2875. * because it is sized independant of architecture. Unlike the other zones,
  2876. * the starting point for ZONE_MOVABLE is not fixed. It may be different
  2877. * in each node depending on the size of each node and how evenly kernelcore
  2878. * is distributed. This helper function adjusts the zone ranges
  2879. * provided by the architecture for a given node by using the end of the
  2880. * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
  2881. * zones within a node are in order of monotonic increases memory addresses
  2882. */
  2883. static void __meminit adjust_zone_range_for_zone_movable(int nid,
  2884. unsigned long zone_type,
  2885. unsigned long node_start_pfn,
  2886. unsigned long node_end_pfn,
  2887. unsigned long *zone_start_pfn,
  2888. unsigned long *zone_end_pfn)
  2889. {
  2890. /* Only adjust if ZONE_MOVABLE is on this node */
  2891. if (zone_movable_pfn[nid]) {
  2892. /* Size ZONE_MOVABLE */
  2893. if (zone_type == ZONE_MOVABLE) {
  2894. *zone_start_pfn = zone_movable_pfn[nid];
  2895. *zone_end_pfn = min(node_end_pfn,
  2896. arch_zone_highest_possible_pfn[movable_zone]);
  2897. /* Adjust for ZONE_MOVABLE starting within this range */
  2898. } else if (*zone_start_pfn < zone_movable_pfn[nid] &&
  2899. *zone_end_pfn > zone_movable_pfn[nid]) {
  2900. *zone_end_pfn = zone_movable_pfn[nid];
  2901. /* Check if this whole range is within ZONE_MOVABLE */
  2902. } else if (*zone_start_pfn >= zone_movable_pfn[nid])
  2903. *zone_start_pfn = *zone_end_pfn;
  2904. }
  2905. }
  2906. /*
  2907. * Return the number of pages a zone spans in a node, including holes
  2908. * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
  2909. */
  2910. static unsigned long __meminit zone_spanned_pages_in_node(int nid,
  2911. unsigned long zone_type,
  2912. unsigned long *ignored)
  2913. {
  2914. unsigned long node_start_pfn, node_end_pfn;
  2915. unsigned long zone_start_pfn, zone_end_pfn;
  2916. /* Get the start and end of the node and zone */
  2917. get_pfn_range_for_nid(nid, &node_start_pfn, &node_end_pfn);
  2918. zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type];
  2919. zone_end_pfn = arch_zone_highest_possible_pfn[zone_type];
  2920. adjust_zone_range_for_zone_movable(nid, zone_type,
  2921. node_start_pfn, node_end_pfn,
  2922. &zone_start_pfn, &zone_end_pfn);
  2923. /* Check that this node has pages within the zone's required range */
  2924. if (zone_end_pfn < node_start_pfn || zone_start_pfn > node_end_pfn)
  2925. return 0;
  2926. /* Move the zone boundaries inside the node if necessary */
  2927. zone_end_pfn = min(zone_end_pfn, node_end_pfn);
  2928. zone_start_pfn = max(zone_start_pfn, node_start_pfn);
  2929. /* Return the spanned pages */
  2930. return zone_end_pfn - zone_start_pfn;
  2931. }
  2932. /*
  2933. * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
  2934. * then all holes in the requested range will be accounted for.
  2935. */
  2936. static unsigned long __meminit __absent_pages_in_range(int nid,
  2937. unsigned long range_start_pfn,
  2938. unsigned long range_end_pfn)
  2939. {
  2940. int i = 0;
  2941. unsigned long prev_end_pfn = 0, hole_pages = 0;
  2942. unsigned long start_pfn;
  2943. /* Find the end_pfn of the first active range of pfns in the node */
  2944. i = first_active_region_index_in_nid(nid);
  2945. if (i == -1)
  2946. return 0;
  2947. prev_end_pfn = min(early_node_map[i].start_pfn, range_end_pfn);
  2948. /* Account for ranges before physical memory on this node */
  2949. if (early_node_map[i].start_pfn > range_start_pfn)
  2950. hole_pages = prev_end_pfn - range_start_pfn;
  2951. /* Find all holes for the zone within the node */
  2952. for (; i != -1; i = next_active_region_index_in_nid(i, nid)) {
  2953. /* No need to continue if prev_end_pfn is outside the zone */
  2954. if (prev_end_pfn >= range_end_pfn)
  2955. break;
  2956. /* Make sure the end of the zone is not within the hole */
  2957. start_pfn = min(early_node_map[i].start_pfn, range_end_pfn);
  2958. prev_end_pfn = max(prev_end_pfn, range_start_pfn);
  2959. /* Update the hole size cound and move on */
  2960. if (start_pfn > range_start_pfn) {
  2961. BUG_ON(prev_end_pfn > start_pfn);
  2962. hole_pages += start_pfn - prev_end_pfn;
  2963. }
  2964. prev_end_pfn = early_node_map[i].end_pfn;
  2965. }
  2966. /* Account for ranges past physical memory on this node */
  2967. if (range_end_pfn > prev_end_pfn)
  2968. hole_pages += range_end_pfn -
  2969. max(range_start_pfn, prev_end_pfn);
  2970. return hole_pages;
  2971. }
  2972. /**
  2973. * absent_pages_in_range - Return number of page frames in holes within a range
  2974. * @start_pfn: The start PFN to start searching for holes
  2975. * @end_pfn: The end PFN to stop searching for holes
  2976. *
  2977. * It returns the number of pages frames in memory holes within a range.
  2978. */
  2979. unsigned long __init absent_pages_in_range(unsigned long start_pfn,
  2980. unsigned long end_pfn)
  2981. {
  2982. return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
  2983. }
  2984. /* Return the number of page frames in holes in a zone on a node */
  2985. static unsigned long __meminit zone_absent_pages_in_node(int nid,
  2986. unsigned long zone_type,
  2987. unsigned long *ignored)
  2988. {
  2989. unsigned long node_start_pfn, node_end_pfn;
  2990. unsigned long zone_start_pfn, zone_end_pfn;
  2991. get_pfn_range_for_nid(nid, &node_start_pfn, &node_end_pfn);
  2992. zone_start_pfn = max(arch_zone_lowest_possible_pfn[zone_type],
  2993. node_start_pfn);
  2994. zone_end_pfn = min(arch_zone_highest_possible_pfn[zone_type],
  2995. node_end_pfn);
  2996. adjust_zone_range_for_zone_movable(nid, zone_type,
  2997. node_start_pfn, node_end_pfn,
  2998. &zone_start_pfn, &zone_end_pfn);
  2999. return __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
  3000. }
  3001. #else
  3002. static inline unsigned long __meminit zone_spanned_pages_in_node(int nid,
  3003. unsigned long zone_type,
  3004. unsigned long *zones_size)
  3005. {
  3006. return zones_size[zone_type];
  3007. }
  3008. static inline unsigned long __meminit zone_absent_pages_in_node(int nid,
  3009. unsigned long zone_type,
  3010. unsigned long *zholes_size)
  3011. {
  3012. if (!zholes_size)
  3013. return 0;
  3014. return zholes_size[zone_type];
  3015. }
  3016. #endif
  3017. static void __meminit calculate_node_totalpages(struct pglist_data *pgdat,
  3018. unsigned long *zones_size, unsigned long *zholes_size)
  3019. {
  3020. unsigned long realtotalpages, totalpages = 0;
  3021. enum zone_type i;
  3022. for (i = 0; i < MAX_NR_ZONES; i++)
  3023. totalpages += zone_spanned_pages_in_node(pgdat->node_id, i,
  3024. zones_size);
  3025. pgdat->node_spanned_pages = totalpages;
  3026. realtotalpages = totalpages;
  3027. for (i = 0; i < MAX_NR_ZONES; i++)
  3028. realtotalpages -=
  3029. zone_absent_pages_in_node(pgdat->node_id, i,
  3030. zholes_size);
  3031. pgdat->node_present_pages = realtotalpages;
  3032. printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id,
  3033. realtotalpages);
  3034. }
  3035. #ifndef CONFIG_SPARSEMEM
  3036. /*
  3037. * Calculate the size of the zone->blockflags rounded to an unsigned long
  3038. * Start by making sure zonesize is a multiple of pageblock_order by rounding
  3039. * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
  3040. * round what is now in bits to nearest long in bits, then return it in
  3041. * bytes.
  3042. */
  3043. static unsigned long __init usemap_size(unsigned long zonesize)
  3044. {
  3045. unsigned long usemapsize;
  3046. usemapsize = roundup(zonesize, pageblock_nr_pages);
  3047. usemapsize = usemapsize >> pageblock_order;
  3048. usemapsize *= NR_PAGEBLOCK_BITS;
  3049. usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
  3050. return usemapsize / 8;
  3051. }
  3052. static void __init setup_usemap(struct pglist_data *pgdat,
  3053. struct zone *zone, unsigned long zonesize)
  3054. {
  3055. unsigned long usemapsize = usemap_size(zonesize);
  3056. zone->pageblock_flags = NULL;
  3057. if (usemapsize)
  3058. zone->pageblock_flags = alloc_bootmem_node(pgdat, usemapsize);
  3059. }
  3060. #else
  3061. static void inline setup_usemap(struct pglist_data *pgdat,
  3062. struct zone *zone, unsigned long zonesize) {}
  3063. #endif /* CONFIG_SPARSEMEM */
  3064. #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
  3065. /* Return a sensible default order for the pageblock size. */
  3066. static inline int pageblock_default_order(void)
  3067. {
  3068. if (HPAGE_SHIFT > PAGE_SHIFT)
  3069. return HUGETLB_PAGE_ORDER;
  3070. return MAX_ORDER-1;
  3071. }
  3072. /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
  3073. static inline void __init set_pageblock_order(unsigned int order)
  3074. {
  3075. /* Check that pageblock_nr_pages has not already been setup */
  3076. if (pageblock_order)
  3077. return;
  3078. /*
  3079. * Assume the largest contiguous order of interest is a huge page.
  3080. * This value may be variable depending on boot parameters on IA64
  3081. */
  3082. pageblock_order = order;
  3083. }
  3084. #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
  3085. /*
  3086. * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
  3087. * and pageblock_default_order() are unused as pageblock_order is set
  3088. * at compile-time. See include/linux/pageblock-flags.h for the values of
  3089. * pageblock_order based on the kernel config
  3090. */
  3091. static inline int pageblock_default_order(unsigned int order)
  3092. {
  3093. return MAX_ORDER-1;
  3094. }
  3095. #define set_pageblock_order(x) do {} while (0)
  3096. #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
  3097. /*
  3098. * Set up the zone data structures:
  3099. * - mark all pages reserved
  3100. * - mark all memory queues empty
  3101. * - clear the memory bitmaps
  3102. */
  3103. static void __paginginit free_area_init_core(struct pglist_data *pgdat,
  3104. unsigned long *zones_size, unsigned long *zholes_size)
  3105. {
  3106. enum zone_type j;
  3107. int nid = pgdat->node_id;
  3108. unsigned long zone_start_pfn = pgdat->node_start_pfn;
  3109. int ret;
  3110. pgdat_resize_init(pgdat);
  3111. pgdat->nr_zones = 0;
  3112. init_waitqueue_head(&pgdat->kswapd_wait);
  3113. pgdat->kswapd_max_order = 0;
  3114. pgdat_page_cgroup_init(pgdat);
  3115. for (j = 0; j < MAX_NR_ZONES; j++) {
  3116. struct zone *zone = pgdat->node_zones + j;
  3117. unsigned long size, realsize, memmap_pages;
  3118. enum lru_list l;
  3119. size = zone_spanned_pages_in_node(nid, j, zones_size);
  3120. realsize = size - zone_absent_pages_in_node(nid, j,
  3121. zholes_size);
  3122. /*
  3123. * Adjust realsize so that it accounts for how much memory
  3124. * is used by this zone for memmap. This affects the watermark
  3125. * and per-cpu initialisations
  3126. */
  3127. memmap_pages =
  3128. PAGE_ALIGN(size * sizeof(struct page)) >> PAGE_SHIFT;
  3129. if (realsize >= memmap_pages) {
  3130. realsize -= memmap_pages;
  3131. if (memmap_pages)
  3132. printk(KERN_DEBUG
  3133. " %s zone: %lu pages used for memmap\n",
  3134. zone_names[j], memmap_pages);
  3135. } else
  3136. printk(KERN_WARNING
  3137. " %s zone: %lu pages exceeds realsize %lu\n",
  3138. zone_names[j], memmap_pages, realsize);
  3139. /* Account for reserved pages */
  3140. if (j == 0 && realsize > dma_reserve) {
  3141. realsize -= dma_reserve;
  3142. printk(KERN_DEBUG " %s zone: %lu pages reserved\n",
  3143. zone_names[0], dma_reserve);
  3144. }
  3145. if (!is_highmem_idx(j))
  3146. nr_kernel_pages += realsize;
  3147. nr_all_pages += realsize;
  3148. zone->spanned_pages = size;
  3149. zone->present_pages = realsize;
  3150. #ifdef CONFIG_NUMA
  3151. zone->node = nid;
  3152. zone->min_unmapped_pages = (realsize*sysctl_min_unmapped_ratio)
  3153. / 100;
  3154. zone->min_slab_pages = (realsize * sysctl_min_slab_ratio) / 100;
  3155. #endif
  3156. zone->name = zone_names[j];
  3157. spin_lock_init(&zone->lock);
  3158. spin_lock_init(&zone->lru_lock);
  3159. zone_seqlock_init(zone);
  3160. zone->zone_pgdat = pgdat;
  3161. zone->prev_priority = DEF_PRIORITY;
  3162. zone_pcp_init(zone);
  3163. for_each_lru(l) {
  3164. INIT_LIST_HEAD(&zone->lru[l].list);
  3165. zone->lru[l].nr_saved_scan = 0;
  3166. }
  3167. zone->reclaim_stat.recent_rotated[0] = 0;
  3168. zone->reclaim_stat.recent_rotated[1] = 0;
  3169. zone->reclaim_stat.recent_scanned[0] = 0;
  3170. zone->reclaim_stat.recent_scanned[1] = 0;
  3171. zap_zone_vm_stats(zone);
  3172. zone->flags = 0;
  3173. if (!size)
  3174. continue;
  3175. set_pageblock_order(pageblock_default_order());
  3176. setup_usemap(pgdat, zone, size);
  3177. ret = init_currently_empty_zone(zone, zone_start_pfn,
  3178. size, MEMMAP_EARLY);
  3179. BUG_ON(ret);
  3180. memmap_init(size, nid, j, zone_start_pfn);
  3181. zone_start_pfn += size;
  3182. }
  3183. }
  3184. static void __init_refok alloc_node_mem_map(struct pglist_data *pgdat)
  3185. {
  3186. /* Skip empty nodes */
  3187. if (!pgdat->node_spanned_pages)
  3188. return;
  3189. #ifdef CONFIG_FLAT_NODE_MEM_MAP
  3190. /* ia64 gets its own node_mem_map, before this, without bootmem */
  3191. if (!pgdat->node_mem_map) {
  3192. unsigned long size, start, end;
  3193. struct page *map;
  3194. /*
  3195. * The zone's endpoints aren't required to be MAX_ORDER
  3196. * aligned but the node_mem_map endpoints must be in order
  3197. * for the buddy allocator to function correctly.
  3198. */
  3199. start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
  3200. end = pgdat->node_start_pfn + pgdat->node_spanned_pages;
  3201. end = ALIGN(end, MAX_ORDER_NR_PAGES);
  3202. size = (end - start) * sizeof(struct page);
  3203. map = alloc_remap(pgdat->node_id, size);
  3204. if (!map)
  3205. map = alloc_bootmem_node(pgdat, size);
  3206. pgdat->node_mem_map = map + (pgdat->node_start_pfn - start);
  3207. }
  3208. #ifndef CONFIG_NEED_MULTIPLE_NODES
  3209. /*
  3210. * With no DISCONTIG, the global mem_map is just set as node 0's
  3211. */
  3212. if (pgdat == NODE_DATA(0)) {
  3213. mem_map = NODE_DATA(0)->node_mem_map;
  3214. #ifdef CONFIG_ARCH_POPULATES_NODE_MAP
  3215. if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
  3216. mem_map -= (pgdat->node_start_pfn - ARCH_PFN_OFFSET);
  3217. #endif /* CONFIG_ARCH_POPULATES_NODE_MAP */
  3218. }
  3219. #endif
  3220. #endif /* CONFIG_FLAT_NODE_MEM_MAP */
  3221. }
  3222. void __paginginit free_area_init_node(int nid, unsigned long *zones_size,
  3223. unsigned long node_start_pfn, unsigned long *zholes_size)
  3224. {
  3225. pg_data_t *pgdat = NODE_DATA(nid);
  3226. pgdat->node_id = nid;
  3227. pgdat->node_start_pfn = node_start_pfn;
  3228. calculate_node_totalpages(pgdat, zones_size, zholes_size);
  3229. alloc_node_mem_map(pgdat);
  3230. #ifdef CONFIG_FLAT_NODE_MEM_MAP
  3231. printk(KERN_DEBUG "free_area_init_node: node %d, pgdat %08lx, node_mem_map %08lx\n",
  3232. nid, (unsigned long)pgdat,
  3233. (unsigned long)pgdat->node_mem_map);
  3234. #endif
  3235. free_area_init_core(pgdat, zones_size, zholes_size);
  3236. }
  3237. #ifdef CONFIG_ARCH_POPULATES_NODE_MAP
  3238. #if MAX_NUMNODES > 1
  3239. /*
  3240. * Figure out the number of possible node ids.
  3241. */
  3242. static void __init setup_nr_node_ids(void)
  3243. {
  3244. unsigned int node;
  3245. unsigned int highest = 0;
  3246. for_each_node_mask(node, node_possible_map)
  3247. highest = node;
  3248. nr_node_ids = highest + 1;
  3249. }
  3250. #else
  3251. static inline void setup_nr_node_ids(void)
  3252. {
  3253. }
  3254. #endif
  3255. /**
  3256. * add_active_range - Register a range of PFNs backed by physical memory
  3257. * @nid: The node ID the range resides on
  3258. * @start_pfn: The start PFN of the available physical memory
  3259. * @end_pfn: The end PFN of the available physical memory
  3260. *
  3261. * These ranges are stored in an early_node_map[] and later used by
  3262. * free_area_init_nodes() to calculate zone sizes and holes. If the
  3263. * range spans a memory hole, it is up to the architecture to ensure
  3264. * the memory is not freed by the bootmem allocator. If possible
  3265. * the range being registered will be merged with existing ranges.
  3266. */
  3267. void __init add_active_range(unsigned int nid, unsigned long start_pfn,
  3268. unsigned long end_pfn)
  3269. {
  3270. int i;
  3271. mminit_dprintk(MMINIT_TRACE, "memory_register",
  3272. "Entering add_active_range(%d, %#lx, %#lx) "
  3273. "%d entries of %d used\n",
  3274. nid, start_pfn, end_pfn,
  3275. nr_nodemap_entries, MAX_ACTIVE_REGIONS);
  3276. mminit_validate_memmodel_limits(&start_pfn, &end_pfn);
  3277. /* Merge with existing active regions if possible */
  3278. for (i = 0; i < nr_nodemap_entries; i++) {
  3279. if (early_node_map[i].nid != nid)
  3280. continue;
  3281. /* Skip if an existing region covers this new one */
  3282. if (start_pfn >= early_node_map[i].start_pfn &&
  3283. end_pfn <= early_node_map[i].end_pfn)
  3284. return;
  3285. /* Merge forward if suitable */
  3286. if (start_pfn <= early_node_map[i].end_pfn &&
  3287. end_pfn > early_node_map[i].end_pfn) {
  3288. early_node_map[i].end_pfn = end_pfn;
  3289. return;
  3290. }
  3291. /* Merge backward if suitable */
  3292. if (start_pfn < early_node_map[i].end_pfn &&
  3293. end_pfn >= early_node_map[i].start_pfn) {
  3294. early_node_map[i].start_pfn = start_pfn;
  3295. return;
  3296. }
  3297. }
  3298. /* Check that early_node_map is large enough */
  3299. if (i >= MAX_ACTIVE_REGIONS) {
  3300. printk(KERN_CRIT "More than %d memory regions, truncating\n",
  3301. MAX_ACTIVE_REGIONS);
  3302. return;
  3303. }
  3304. early_node_map[i].nid = nid;
  3305. early_node_map[i].start_pfn = start_pfn;
  3306. early_node_map[i].end_pfn = end_pfn;
  3307. nr_nodemap_entries = i + 1;
  3308. }
  3309. /**
  3310. * remove_active_range - Shrink an existing registered range of PFNs
  3311. * @nid: The node id the range is on that should be shrunk
  3312. * @start_pfn: The new PFN of the range
  3313. * @end_pfn: The new PFN of the range
  3314. *
  3315. * i386 with NUMA use alloc_remap() to store a node_mem_map on a local node.
  3316. * The map is kept near the end physical page range that has already been
  3317. * registered. This function allows an arch to shrink an existing registered
  3318. * range.
  3319. */
  3320. void __init remove_active_range(unsigned int nid, unsigned long start_pfn,
  3321. unsigned long end_pfn)
  3322. {
  3323. int i, j;
  3324. int removed = 0;
  3325. printk(KERN_DEBUG "remove_active_range (%d, %lu, %lu)\n",
  3326. nid, start_pfn, end_pfn);
  3327. /* Find the old active region end and shrink */
  3328. for_each_active_range_index_in_nid(i, nid) {
  3329. if (early_node_map[i].start_pfn >= start_pfn &&
  3330. early_node_map[i].end_pfn <= end_pfn) {
  3331. /* clear it */
  3332. early_node_map[i].start_pfn = 0;
  3333. early_node_map[i].end_pfn = 0;
  3334. removed = 1;
  3335. continue;
  3336. }
  3337. if (early_node_map[i].start_pfn < start_pfn &&
  3338. early_node_map[i].end_pfn > start_pfn) {
  3339. unsigned long temp_end_pfn = early_node_map[i].end_pfn;
  3340. early_node_map[i].end_pfn = start_pfn;
  3341. if (temp_end_pfn > end_pfn)
  3342. add_active_range(nid, end_pfn, temp_end_pfn);
  3343. continue;
  3344. }
  3345. if (early_node_map[i].start_pfn >= start_pfn &&
  3346. early_node_map[i].end_pfn > end_pfn &&
  3347. early_node_map[i].start_pfn < end_pfn) {
  3348. early_node_map[i].start_pfn = end_pfn;
  3349. continue;
  3350. }
  3351. }
  3352. if (!removed)
  3353. return;
  3354. /* remove the blank ones */
  3355. for (i = nr_nodemap_entries - 1; i > 0; i--) {
  3356. if (early_node_map[i].nid != nid)
  3357. continue;
  3358. if (early_node_map[i].end_pfn)
  3359. continue;
  3360. /* we found it, get rid of it */
  3361. for (j = i; j < nr_nodemap_entries - 1; j++)
  3362. memcpy(&early_node_map[j], &early_node_map[j+1],
  3363. sizeof(early_node_map[j]));
  3364. j = nr_nodemap_entries - 1;
  3365. memset(&early_node_map[j], 0, sizeof(early_node_map[j]));
  3366. nr_nodemap_entries--;
  3367. }
  3368. }
  3369. /**
  3370. * remove_all_active_ranges - Remove all currently registered regions
  3371. *
  3372. * During discovery, it may be found that a table like SRAT is invalid
  3373. * and an alternative discovery method must be used. This function removes
  3374. * all currently registered regions.
  3375. */
  3376. void __init remove_all_active_ranges(void)
  3377. {
  3378. memset(early_node_map, 0, sizeof(early_node_map));
  3379. nr_nodemap_entries = 0;
  3380. }
  3381. /* Compare two active node_active_regions */
  3382. static int __init cmp_node_active_region(const void *a, const void *b)
  3383. {
  3384. struct node_active_region *arange = (struct node_active_region *)a;
  3385. struct node_active_region *brange = (struct node_active_region *)b;
  3386. /* Done this way to avoid overflows */
  3387. if (arange->start_pfn > brange->start_pfn)
  3388. return 1;
  3389. if (arange->start_pfn < brange->start_pfn)
  3390. return -1;
  3391. return 0;
  3392. }
  3393. /* sort the node_map by start_pfn */
  3394. static void __init sort_node_map(void)
  3395. {
  3396. sort(early_node_map, (size_t)nr_nodemap_entries,
  3397. sizeof(struct node_active_region),
  3398. cmp_node_active_region, NULL);
  3399. }
  3400. /* Find the lowest pfn for a node */
  3401. static unsigned long __init find_min_pfn_for_node(int nid)
  3402. {
  3403. int i;
  3404. unsigned long min_pfn = ULONG_MAX;
  3405. /* Assuming a sorted map, the first range found has the starting pfn */
  3406. for_each_active_range_index_in_nid(i, nid)
  3407. min_pfn = min(min_pfn, early_node_map[i].start_pfn);
  3408. if (min_pfn == ULONG_MAX) {
  3409. printk(KERN_WARNING
  3410. "Could not find start_pfn for node %d\n", nid);
  3411. return 0;
  3412. }
  3413. return min_pfn;
  3414. }
  3415. /**
  3416. * find_min_pfn_with_active_regions - Find the minimum PFN registered
  3417. *
  3418. * It returns the minimum PFN based on information provided via
  3419. * add_active_range().
  3420. */
  3421. unsigned long __init find_min_pfn_with_active_regions(void)
  3422. {
  3423. return find_min_pfn_for_node(MAX_NUMNODES);
  3424. }
  3425. /*
  3426. * early_calculate_totalpages()
  3427. * Sum pages in active regions for movable zone.
  3428. * Populate N_HIGH_MEMORY for calculating usable_nodes.
  3429. */
  3430. static unsigned long __init early_calculate_totalpages(void)
  3431. {
  3432. int i;
  3433. unsigned long totalpages = 0;
  3434. for (i = 0; i < nr_nodemap_entries; i++) {
  3435. unsigned long pages = early_node_map[i].end_pfn -
  3436. early_node_map[i].start_pfn;
  3437. totalpages += pages;
  3438. if (pages)
  3439. node_set_state(early_node_map[i].nid, N_HIGH_MEMORY);
  3440. }
  3441. return totalpages;
  3442. }
  3443. /*
  3444. * Find the PFN the Movable zone begins in each node. Kernel memory
  3445. * is spread evenly between nodes as long as the nodes have enough
  3446. * memory. When they don't, some nodes will have more kernelcore than
  3447. * others
  3448. */
  3449. static void __init find_zone_movable_pfns_for_nodes(unsigned long *movable_pfn)
  3450. {
  3451. int i, nid;
  3452. unsigned long usable_startpfn;
  3453. unsigned long kernelcore_node, kernelcore_remaining;
  3454. unsigned long totalpages = early_calculate_totalpages();
  3455. int usable_nodes = nodes_weight(node_states[N_HIGH_MEMORY]);
  3456. /*
  3457. * If movablecore was specified, calculate what size of
  3458. * kernelcore that corresponds so that memory usable for
  3459. * any allocation type is evenly spread. If both kernelcore
  3460. * and movablecore are specified, then the value of kernelcore
  3461. * will be used for required_kernelcore if it's greater than
  3462. * what movablecore would have allowed.
  3463. */
  3464. if (required_movablecore) {
  3465. unsigned long corepages;
  3466. /*
  3467. * Round-up so that ZONE_MOVABLE is at least as large as what
  3468. * was requested by the user
  3469. */
  3470. required_movablecore =
  3471. roundup(required_movablecore, MAX_ORDER_NR_PAGES);
  3472. corepages = totalpages - required_movablecore;
  3473. required_kernelcore = max(required_kernelcore, corepages);
  3474. }
  3475. /* If kernelcore was not specified, there is no ZONE_MOVABLE */
  3476. if (!required_kernelcore)
  3477. return;
  3478. /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
  3479. find_usable_zone_for_movable();
  3480. usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
  3481. restart:
  3482. /* Spread kernelcore memory as evenly as possible throughout nodes */
  3483. kernelcore_node = required_kernelcore / usable_nodes;
  3484. for_each_node_state(nid, N_HIGH_MEMORY) {
  3485. /*
  3486. * Recalculate kernelcore_node if the division per node
  3487. * now exceeds what is necessary to satisfy the requested
  3488. * amount of memory for the kernel
  3489. */
  3490. if (required_kernelcore < kernelcore_node)
  3491. kernelcore_node = required_kernelcore / usable_nodes;
  3492. /*
  3493. * As the map is walked, we track how much memory is usable
  3494. * by the kernel using kernelcore_remaining. When it is
  3495. * 0, the rest of the node is usable by ZONE_MOVABLE
  3496. */
  3497. kernelcore_remaining = kernelcore_node;
  3498. /* Go through each range of PFNs within this node */
  3499. for_each_active_range_index_in_nid(i, nid) {
  3500. unsigned long start_pfn, end_pfn;
  3501. unsigned long size_pages;
  3502. start_pfn = max(early_node_map[i].start_pfn,
  3503. zone_movable_pfn[nid]);
  3504. end_pfn = early_node_map[i].end_pfn;
  3505. if (start_pfn >= end_pfn)
  3506. continue;
  3507. /* Account for what is only usable for kernelcore */
  3508. if (start_pfn < usable_startpfn) {
  3509. unsigned long kernel_pages;
  3510. kernel_pages = min(end_pfn, usable_startpfn)
  3511. - start_pfn;
  3512. kernelcore_remaining -= min(kernel_pages,
  3513. kernelcore_remaining);
  3514. required_kernelcore -= min(kernel_pages,
  3515. required_kernelcore);
  3516. /* Continue if range is now fully accounted */
  3517. if (end_pfn <= usable_startpfn) {
  3518. /*
  3519. * Push zone_movable_pfn to the end so
  3520. * that if we have to rebalance
  3521. * kernelcore across nodes, we will
  3522. * not double account here
  3523. */
  3524. zone_movable_pfn[nid] = end_pfn;
  3525. continue;
  3526. }
  3527. start_pfn = usable_startpfn;
  3528. }
  3529. /*
  3530. * The usable PFN range for ZONE_MOVABLE is from
  3531. * start_pfn->end_pfn. Calculate size_pages as the
  3532. * number of pages used as kernelcore
  3533. */
  3534. size_pages = end_pfn - start_pfn;
  3535. if (size_pages > kernelcore_remaining)
  3536. size_pages = kernelcore_remaining;
  3537. zone_movable_pfn[nid] = start_pfn + size_pages;
  3538. /*
  3539. * Some kernelcore has been met, update counts and
  3540. * break if the kernelcore for this node has been
  3541. * satisified
  3542. */
  3543. required_kernelcore -= min(required_kernelcore,
  3544. size_pages);
  3545. kernelcore_remaining -= size_pages;
  3546. if (!kernelcore_remaining)
  3547. break;
  3548. }
  3549. }
  3550. /*
  3551. * If there is still required_kernelcore, we do another pass with one
  3552. * less node in the count. This will push zone_movable_pfn[nid] further
  3553. * along on the nodes that still have memory until kernelcore is
  3554. * satisified
  3555. */
  3556. usable_nodes--;
  3557. if (usable_nodes && required_kernelcore > usable_nodes)
  3558. goto restart;
  3559. /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
  3560. for (nid = 0; nid < MAX_NUMNODES; nid++)
  3561. zone_movable_pfn[nid] =
  3562. roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
  3563. }
  3564. /* Any regular memory on that node ? */
  3565. static void check_for_regular_memory(pg_data_t *pgdat)
  3566. {
  3567. #ifdef CONFIG_HIGHMEM
  3568. enum zone_type zone_type;
  3569. for (zone_type = 0; zone_type <= ZONE_NORMAL; zone_type++) {
  3570. struct zone *zone = &pgdat->node_zones[zone_type];
  3571. if (zone->present_pages)
  3572. node_set_state(zone_to_nid(zone), N_NORMAL_MEMORY);
  3573. }
  3574. #endif
  3575. }
  3576. /**
  3577. * free_area_init_nodes - Initialise all pg_data_t and zone data
  3578. * @max_zone_pfn: an array of max PFNs for each zone
  3579. *
  3580. * This will call free_area_init_node() for each active node in the system.
  3581. * Using the page ranges provided by add_active_range(), the size of each
  3582. * zone in each node and their holes is calculated. If the maximum PFN
  3583. * between two adjacent zones match, it is assumed that the zone is empty.
  3584. * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
  3585. * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
  3586. * starts where the previous one ended. For example, ZONE_DMA32 starts
  3587. * at arch_max_dma_pfn.
  3588. */
  3589. void __init free_area_init_nodes(unsigned long *max_zone_pfn)
  3590. {
  3591. unsigned long nid;
  3592. int i;
  3593. /* Sort early_node_map as initialisation assumes it is sorted */
  3594. sort_node_map();
  3595. /* Record where the zone boundaries are */
  3596. memset(arch_zone_lowest_possible_pfn, 0,
  3597. sizeof(arch_zone_lowest_possible_pfn));
  3598. memset(arch_zone_highest_possible_pfn, 0,
  3599. sizeof(arch_zone_highest_possible_pfn));
  3600. arch_zone_lowest_possible_pfn[0] = find_min_pfn_with_active_regions();
  3601. arch_zone_highest_possible_pfn[0] = max_zone_pfn[0];
  3602. for (i = 1; i < MAX_NR_ZONES; i++) {
  3603. if (i == ZONE_MOVABLE)
  3604. continue;
  3605. arch_zone_lowest_possible_pfn[i] =
  3606. arch_zone_highest_possible_pfn[i-1];
  3607. arch_zone_highest_possible_pfn[i] =
  3608. max(max_zone_pfn[i], arch_zone_lowest_possible_pfn[i]);
  3609. }
  3610. arch_zone_lowest_possible_pfn[ZONE_MOVABLE] = 0;
  3611. arch_zone_highest_possible_pfn[ZONE_MOVABLE] = 0;
  3612. /* Find the PFNs that ZONE_MOVABLE begins at in each node */
  3613. memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
  3614. find_zone_movable_pfns_for_nodes(zone_movable_pfn);
  3615. /* Print out the zone ranges */
  3616. printk("Zone PFN ranges:\n");
  3617. for (i = 0; i < MAX_NR_ZONES; i++) {
  3618. if (i == ZONE_MOVABLE)
  3619. continue;
  3620. printk(" %-8s %0#10lx -> %0#10lx\n",
  3621. zone_names[i],
  3622. arch_zone_lowest_possible_pfn[i],
  3623. arch_zone_highest_possible_pfn[i]);
  3624. }
  3625. /* Print out the PFNs ZONE_MOVABLE begins at in each node */
  3626. printk("Movable zone start PFN for each node\n");
  3627. for (i = 0; i < MAX_NUMNODES; i++) {
  3628. if (zone_movable_pfn[i])
  3629. printk(" Node %d: %lu\n", i, zone_movable_pfn[i]);
  3630. }
  3631. /* Print out the early_node_map[] */
  3632. printk("early_node_map[%d] active PFN ranges\n", nr_nodemap_entries);
  3633. for (i = 0; i < nr_nodemap_entries; i++)
  3634. printk(" %3d: %0#10lx -> %0#10lx\n", early_node_map[i].nid,
  3635. early_node_map[i].start_pfn,
  3636. early_node_map[i].end_pfn);
  3637. /* Initialise every node */
  3638. mminit_verify_pageflags_layout();
  3639. setup_nr_node_ids();
  3640. for_each_online_node(nid) {
  3641. pg_data_t *pgdat = NODE_DATA(nid);
  3642. free_area_init_node(nid, NULL,
  3643. find_min_pfn_for_node(nid), NULL);
  3644. /* Any memory on that node */
  3645. if (pgdat->node_present_pages)
  3646. node_set_state(nid, N_HIGH_MEMORY);
  3647. check_for_regular_memory(pgdat);
  3648. }
  3649. }
  3650. static int __init cmdline_parse_core(char *p, unsigned long *core)
  3651. {
  3652. unsigned long long coremem;
  3653. if (!p)
  3654. return -EINVAL;
  3655. coremem = memparse(p, &p);
  3656. *core = coremem >> PAGE_SHIFT;
  3657. /* Paranoid check that UL is enough for the coremem value */
  3658. WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
  3659. return 0;
  3660. }
  3661. /*
  3662. * kernelcore=size sets the amount of memory for use for allocations that
  3663. * cannot be reclaimed or migrated.
  3664. */
  3665. static int __init cmdline_parse_kernelcore(char *p)
  3666. {
  3667. return cmdline_parse_core(p, &required_kernelcore);
  3668. }
  3669. /*
  3670. * movablecore=size sets the amount of memory for use for allocations that
  3671. * can be reclaimed or migrated.
  3672. */
  3673. static int __init cmdline_parse_movablecore(char *p)
  3674. {
  3675. return cmdline_parse_core(p, &required_movablecore);
  3676. }
  3677. early_param("kernelcore", cmdline_parse_kernelcore);
  3678. early_param("movablecore", cmdline_parse_movablecore);
  3679. #endif /* CONFIG_ARCH_POPULATES_NODE_MAP */
  3680. /**
  3681. * set_dma_reserve - set the specified number of pages reserved in the first zone
  3682. * @new_dma_reserve: The number of pages to mark reserved
  3683. *
  3684. * The per-cpu batchsize and zone watermarks are determined by present_pages.
  3685. * In the DMA zone, a significant percentage may be consumed by kernel image
  3686. * and other unfreeable allocations which can skew the watermarks badly. This
  3687. * function may optionally be used to account for unfreeable pages in the
  3688. * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
  3689. * smaller per-cpu batchsize.
  3690. */
  3691. void __init set_dma_reserve(unsigned long new_dma_reserve)
  3692. {
  3693. dma_reserve = new_dma_reserve;
  3694. }
  3695. #ifndef CONFIG_NEED_MULTIPLE_NODES
  3696. struct pglist_data __refdata contig_page_data = { .bdata = &bootmem_node_data[0] };
  3697. EXPORT_SYMBOL(contig_page_data);
  3698. #endif
  3699. void __init free_area_init(unsigned long *zones_size)
  3700. {
  3701. free_area_init_node(0, zones_size,
  3702. __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL);
  3703. }
  3704. static int page_alloc_cpu_notify(struct notifier_block *self,
  3705. unsigned long action, void *hcpu)
  3706. {
  3707. int cpu = (unsigned long)hcpu;
  3708. if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
  3709. drain_pages(cpu);
  3710. /*
  3711. * Spill the event counters of the dead processor
  3712. * into the current processors event counters.
  3713. * This artificially elevates the count of the current
  3714. * processor.
  3715. */
  3716. vm_events_fold_cpu(cpu);
  3717. /*
  3718. * Zero the differential counters of the dead processor
  3719. * so that the vm statistics are consistent.
  3720. *
  3721. * This is only okay since the processor is dead and cannot
  3722. * race with what we are doing.
  3723. */
  3724. refresh_cpu_vm_stats(cpu);
  3725. }
  3726. return NOTIFY_OK;
  3727. }
  3728. void __init page_alloc_init(void)
  3729. {
  3730. hotcpu_notifier(page_alloc_cpu_notify, 0);
  3731. }
  3732. /*
  3733. * calculate_totalreserve_pages - called when sysctl_lower_zone_reserve_ratio
  3734. * or min_free_kbytes changes.
  3735. */
  3736. static void calculate_totalreserve_pages(void)
  3737. {
  3738. struct pglist_data *pgdat;
  3739. unsigned long reserve_pages = 0;
  3740. enum zone_type i, j;
  3741. for_each_online_pgdat(pgdat) {
  3742. for (i = 0; i < MAX_NR_ZONES; i++) {
  3743. struct zone *zone = pgdat->node_zones + i;
  3744. unsigned long max = 0;
  3745. /* Find valid and maximum lowmem_reserve in the zone */
  3746. for (j = i; j < MAX_NR_ZONES; j++) {
  3747. if (zone->lowmem_reserve[j] > max)
  3748. max = zone->lowmem_reserve[j];
  3749. }
  3750. /* we treat the high watermark as reserved pages. */
  3751. max += high_wmark_pages(zone);
  3752. if (max > zone->present_pages)
  3753. max = zone->present_pages;
  3754. reserve_pages += max;
  3755. }
  3756. }
  3757. totalreserve_pages = reserve_pages;
  3758. }
  3759. /*
  3760. * setup_per_zone_lowmem_reserve - called whenever
  3761. * sysctl_lower_zone_reserve_ratio changes. Ensures that each zone
  3762. * has a correct pages reserved value, so an adequate number of
  3763. * pages are left in the zone after a successful __alloc_pages().
  3764. */
  3765. static void setup_per_zone_lowmem_reserve(void)
  3766. {
  3767. struct pglist_data *pgdat;
  3768. enum zone_type j, idx;
  3769. for_each_online_pgdat(pgdat) {
  3770. for (j = 0; j < MAX_NR_ZONES; j++) {
  3771. struct zone *zone = pgdat->node_zones + j;
  3772. unsigned long present_pages = zone->present_pages;
  3773. zone->lowmem_reserve[j] = 0;
  3774. idx = j;
  3775. while (idx) {
  3776. struct zone *lower_zone;
  3777. idx--;
  3778. if (sysctl_lowmem_reserve_ratio[idx] < 1)
  3779. sysctl_lowmem_reserve_ratio[idx] = 1;
  3780. lower_zone = pgdat->node_zones + idx;
  3781. lower_zone->lowmem_reserve[j] = present_pages /
  3782. sysctl_lowmem_reserve_ratio[idx];
  3783. present_pages += lower_zone->present_pages;
  3784. }
  3785. }
  3786. }
  3787. /* update totalreserve_pages */
  3788. calculate_totalreserve_pages();
  3789. }
  3790. /**
  3791. * setup_per_zone_wmarks - called when min_free_kbytes changes
  3792. * or when memory is hot-{added|removed}
  3793. *
  3794. * Ensures that the watermark[min,low,high] values for each zone are set
  3795. * correctly with respect to min_free_kbytes.
  3796. */
  3797. void setup_per_zone_wmarks(void)
  3798. {
  3799. unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
  3800. unsigned long lowmem_pages = 0;
  3801. struct zone *zone;
  3802. unsigned long flags;
  3803. /* Calculate total number of !ZONE_HIGHMEM pages */
  3804. for_each_zone(zone) {
  3805. if (!is_highmem(zone))
  3806. lowmem_pages += zone->present_pages;
  3807. }
  3808. for_each_zone(zone) {
  3809. u64 tmp;
  3810. spin_lock_irqsave(&zone->lock, flags);
  3811. tmp = (u64)pages_min * zone->present_pages;
  3812. do_div(tmp, lowmem_pages);
  3813. if (is_highmem(zone)) {
  3814. /*
  3815. * __GFP_HIGH and PF_MEMALLOC allocations usually don't
  3816. * need highmem pages, so cap pages_min to a small
  3817. * value here.
  3818. *
  3819. * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
  3820. * deltas controls asynch page reclaim, and so should
  3821. * not be capped for highmem.
  3822. */
  3823. int min_pages;
  3824. min_pages = zone->present_pages / 1024;
  3825. if (min_pages < SWAP_CLUSTER_MAX)
  3826. min_pages = SWAP_CLUSTER_MAX;
  3827. if (min_pages > 128)
  3828. min_pages = 128;
  3829. zone->watermark[WMARK_MIN] = min_pages;
  3830. } else {
  3831. /*
  3832. * If it's a lowmem zone, reserve a number of pages
  3833. * proportionate to the zone's size.
  3834. */
  3835. zone->watermark[WMARK_MIN] = tmp;
  3836. }
  3837. zone->watermark[WMARK_LOW] = min_wmark_pages(zone) + (tmp >> 2);
  3838. zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + (tmp >> 1);
  3839. setup_zone_migrate_reserve(zone);
  3840. spin_unlock_irqrestore(&zone->lock, flags);
  3841. }
  3842. /* update totalreserve_pages */
  3843. calculate_totalreserve_pages();
  3844. }
  3845. /**
  3846. * The inactive anon list should be small enough that the VM never has to
  3847. * do too much work, but large enough that each inactive page has a chance
  3848. * to be referenced again before it is swapped out.
  3849. *
  3850. * The inactive_anon ratio is the target ratio of ACTIVE_ANON to
  3851. * INACTIVE_ANON pages on this zone's LRU, maintained by the
  3852. * pageout code. A zone->inactive_ratio of 3 means 3:1 or 25% of
  3853. * the anonymous pages are kept on the inactive list.
  3854. *
  3855. * total target max
  3856. * memory ratio inactive anon
  3857. * -------------------------------------
  3858. * 10MB 1 5MB
  3859. * 100MB 1 50MB
  3860. * 1GB 3 250MB
  3861. * 10GB 10 0.9GB
  3862. * 100GB 31 3GB
  3863. * 1TB 101 10GB
  3864. * 10TB 320 32GB
  3865. */
  3866. void calculate_zone_inactive_ratio(struct zone *zone)
  3867. {
  3868. unsigned int gb, ratio;
  3869. /* Zone size in gigabytes */
  3870. gb = zone->present_pages >> (30 - PAGE_SHIFT);
  3871. if (gb)
  3872. ratio = int_sqrt(10 * gb);
  3873. else
  3874. ratio = 1;
  3875. zone->inactive_ratio = ratio;
  3876. }
  3877. static void __init setup_per_zone_inactive_ratio(void)
  3878. {
  3879. struct zone *zone;
  3880. for_each_zone(zone)
  3881. calculate_zone_inactive_ratio(zone);
  3882. }
  3883. /*
  3884. * Initialise min_free_kbytes.
  3885. *
  3886. * For small machines we want it small (128k min). For large machines
  3887. * we want it large (64MB max). But it is not linear, because network
  3888. * bandwidth does not increase linearly with machine size. We use
  3889. *
  3890. * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy:
  3891. * min_free_kbytes = sqrt(lowmem_kbytes * 16)
  3892. *
  3893. * which yields
  3894. *
  3895. * 16MB: 512k
  3896. * 32MB: 724k
  3897. * 64MB: 1024k
  3898. * 128MB: 1448k
  3899. * 256MB: 2048k
  3900. * 512MB: 2896k
  3901. * 1024MB: 4096k
  3902. * 2048MB: 5792k
  3903. * 4096MB: 8192k
  3904. * 8192MB: 11584k
  3905. * 16384MB: 16384k
  3906. */
  3907. static int __init init_per_zone_wmark_min(void)
  3908. {
  3909. unsigned long lowmem_kbytes;
  3910. lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
  3911. min_free_kbytes = int_sqrt(lowmem_kbytes * 16);
  3912. if (min_free_kbytes < 128)
  3913. min_free_kbytes = 128;
  3914. if (min_free_kbytes > 65536)
  3915. min_free_kbytes = 65536;
  3916. setup_per_zone_wmarks();
  3917. setup_per_zone_lowmem_reserve();
  3918. setup_per_zone_inactive_ratio();
  3919. return 0;
  3920. }
  3921. module_init(init_per_zone_wmark_min)
  3922. /*
  3923. * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so
  3924. * that we can call two helper functions whenever min_free_kbytes
  3925. * changes.
  3926. */
  3927. int min_free_kbytes_sysctl_handler(ctl_table *table, int write,
  3928. struct file *file, void __user *buffer, size_t *length, loff_t *ppos)
  3929. {
  3930. proc_dointvec(table, write, file, buffer, length, ppos);
  3931. if (write)
  3932. setup_per_zone_wmarks();
  3933. return 0;
  3934. }
  3935. #ifdef CONFIG_NUMA
  3936. int sysctl_min_unmapped_ratio_sysctl_handler(ctl_table *table, int write,
  3937. struct file *file, void __user *buffer, size_t *length, loff_t *ppos)
  3938. {
  3939. struct zone *zone;
  3940. int rc;
  3941. rc = proc_dointvec_minmax(table, write, file, buffer, length, ppos);
  3942. if (rc)
  3943. return rc;
  3944. for_each_zone(zone)
  3945. zone->min_unmapped_pages = (zone->present_pages *
  3946. sysctl_min_unmapped_ratio) / 100;
  3947. return 0;
  3948. }
  3949. int sysctl_min_slab_ratio_sysctl_handler(ctl_table *table, int write,
  3950. struct file *file, void __user *buffer, size_t *length, loff_t *ppos)
  3951. {
  3952. struct zone *zone;
  3953. int rc;
  3954. rc = proc_dointvec_minmax(table, write, file, buffer, length, ppos);
  3955. if (rc)
  3956. return rc;
  3957. for_each_zone(zone)
  3958. zone->min_slab_pages = (zone->present_pages *
  3959. sysctl_min_slab_ratio) / 100;
  3960. return 0;
  3961. }
  3962. #endif
  3963. /*
  3964. * lowmem_reserve_ratio_sysctl_handler - just a wrapper around
  3965. * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve()
  3966. * whenever sysctl_lowmem_reserve_ratio changes.
  3967. *
  3968. * The reserve ratio obviously has absolutely no relation with the
  3969. * minimum watermarks. The lowmem reserve ratio can only make sense
  3970. * if in function of the boot time zone sizes.
  3971. */
  3972. int lowmem_reserve_ratio_sysctl_handler(ctl_table *table, int write,
  3973. struct file *file, void __user *buffer, size_t *length, loff_t *ppos)
  3974. {
  3975. proc_dointvec_minmax(table, write, file, buffer, length, ppos);
  3976. setup_per_zone_lowmem_reserve();
  3977. return 0;
  3978. }
  3979. /*
  3980. * percpu_pagelist_fraction - changes the pcp->high for each zone on each
  3981. * cpu. It is the fraction of total pages in each zone that a hot per cpu pagelist
  3982. * can have before it gets flushed back to buddy allocator.
  3983. */
  3984. int percpu_pagelist_fraction_sysctl_handler(ctl_table *table, int write,
  3985. struct file *file, void __user *buffer, size_t *length, loff_t *ppos)
  3986. {
  3987. struct zone *zone;
  3988. unsigned int cpu;
  3989. int ret;
  3990. ret = proc_dointvec_minmax(table, write, file, buffer, length, ppos);
  3991. if (!write || (ret == -EINVAL))
  3992. return ret;
  3993. for_each_zone(zone) {
  3994. for_each_online_cpu(cpu) {
  3995. unsigned long high;
  3996. high = zone->present_pages / percpu_pagelist_fraction;
  3997. setup_pagelist_highmark(zone_pcp(zone, cpu), high);
  3998. }
  3999. }
  4000. return 0;
  4001. }
  4002. int hashdist = HASHDIST_DEFAULT;
  4003. #ifdef CONFIG_NUMA
  4004. static int __init set_hashdist(char *str)
  4005. {
  4006. if (!str)
  4007. return 0;
  4008. hashdist = simple_strtoul(str, &str, 0);
  4009. return 1;
  4010. }
  4011. __setup("hashdist=", set_hashdist);
  4012. #endif
  4013. /*
  4014. * allocate a large system hash table from bootmem
  4015. * - it is assumed that the hash table must contain an exact power-of-2
  4016. * quantity of entries
  4017. * - limit is the number of hash buckets, not the total allocation size
  4018. */
  4019. void *__init alloc_large_system_hash(const char *tablename,
  4020. unsigned long bucketsize,
  4021. unsigned long numentries,
  4022. int scale,
  4023. int flags,
  4024. unsigned int *_hash_shift,
  4025. unsigned int *_hash_mask,
  4026. unsigned long limit)
  4027. {
  4028. unsigned long long max = limit;
  4029. unsigned long log2qty, size;
  4030. void *table = NULL;
  4031. /* allow the kernel cmdline to have a say */
  4032. if (!numentries) {
  4033. /* round applicable memory size up to nearest megabyte */
  4034. numentries = nr_kernel_pages;
  4035. numentries += (1UL << (20 - PAGE_SHIFT)) - 1;
  4036. numentries >>= 20 - PAGE_SHIFT;
  4037. numentries <<= 20 - PAGE_SHIFT;
  4038. /* limit to 1 bucket per 2^scale bytes of low memory */
  4039. if (scale > PAGE_SHIFT)
  4040. numentries >>= (scale - PAGE_SHIFT);
  4041. else
  4042. numentries <<= (PAGE_SHIFT - scale);
  4043. /* Make sure we've got at least a 0-order allocation.. */
  4044. if (unlikely((numentries * bucketsize) < PAGE_SIZE))
  4045. numentries = PAGE_SIZE / bucketsize;
  4046. }
  4047. numentries = roundup_pow_of_two(numentries);
  4048. /* limit allocation size to 1/16 total memory by default */
  4049. if (max == 0) {
  4050. max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
  4051. do_div(max, bucketsize);
  4052. }
  4053. if (numentries > max)
  4054. numentries = max;
  4055. log2qty = ilog2(numentries);
  4056. do {
  4057. size = bucketsize << log2qty;
  4058. if (flags & HASH_EARLY)
  4059. table = alloc_bootmem_nopanic(size);
  4060. else if (hashdist)
  4061. table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL);
  4062. else {
  4063. /*
  4064. * If bucketsize is not a power-of-two, we may free
  4065. * some pages at the end of hash table which
  4066. * alloc_pages_exact() automatically does
  4067. */
  4068. if (get_order(size) < MAX_ORDER)
  4069. table = alloc_pages_exact(size, GFP_ATOMIC);
  4070. }
  4071. } while (!table && size > PAGE_SIZE && --log2qty);
  4072. if (!table)
  4073. panic("Failed to allocate %s hash table\n", tablename);
  4074. printk(KERN_INFO "%s hash table entries: %d (order: %d, %lu bytes)\n",
  4075. tablename,
  4076. (1U << log2qty),
  4077. ilog2(size) - PAGE_SHIFT,
  4078. size);
  4079. if (_hash_shift)
  4080. *_hash_shift = log2qty;
  4081. if (_hash_mask)
  4082. *_hash_mask = (1 << log2qty) - 1;
  4083. /*
  4084. * If hashdist is set, the table allocation is done with __vmalloc()
  4085. * which invokes the kmemleak_alloc() callback. This function may also
  4086. * be called before the slab and kmemleak are initialised when
  4087. * kmemleak simply buffers the request to be executed later
  4088. * (GFP_ATOMIC flag ignored in this case).
  4089. */
  4090. if (!hashdist)
  4091. kmemleak_alloc(table, size, 1, GFP_ATOMIC);
  4092. return table;
  4093. }
  4094. /* Return a pointer to the bitmap storing bits affecting a block of pages */
  4095. static inline unsigned long *get_pageblock_bitmap(struct zone *zone,
  4096. unsigned long pfn)
  4097. {
  4098. #ifdef CONFIG_SPARSEMEM
  4099. return __pfn_to_section(pfn)->pageblock_flags;
  4100. #else
  4101. return zone->pageblock_flags;
  4102. #endif /* CONFIG_SPARSEMEM */
  4103. }
  4104. static inline int pfn_to_bitidx(struct zone *zone, unsigned long pfn)
  4105. {
  4106. #ifdef CONFIG_SPARSEMEM
  4107. pfn &= (PAGES_PER_SECTION-1);
  4108. return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
  4109. #else
  4110. pfn = pfn - zone->zone_start_pfn;
  4111. return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
  4112. #endif /* CONFIG_SPARSEMEM */
  4113. }
  4114. /**
  4115. * get_pageblock_flags_group - Return the requested group of flags for the pageblock_nr_pages block of pages
  4116. * @page: The page within the block of interest
  4117. * @start_bitidx: The first bit of interest to retrieve
  4118. * @end_bitidx: The last bit of interest
  4119. * returns pageblock_bits flags
  4120. */
  4121. unsigned long get_pageblock_flags_group(struct page *page,
  4122. int start_bitidx, int end_bitidx)
  4123. {
  4124. struct zone *zone;
  4125. unsigned long *bitmap;
  4126. unsigned long pfn, bitidx;
  4127. unsigned long flags = 0;
  4128. unsigned long value = 1;
  4129. zone = page_zone(page);
  4130. pfn = page_to_pfn(page);
  4131. bitmap = get_pageblock_bitmap(zone, pfn);
  4132. bitidx = pfn_to_bitidx(zone, pfn);
  4133. for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1)
  4134. if (test_bit(bitidx + start_bitidx, bitmap))
  4135. flags |= value;
  4136. return flags;
  4137. }
  4138. /**
  4139. * set_pageblock_flags_group - Set the requested group of flags for a pageblock_nr_pages block of pages
  4140. * @page: The page within the block of interest
  4141. * @start_bitidx: The first bit of interest
  4142. * @end_bitidx: The last bit of interest
  4143. * @flags: The flags to set
  4144. */
  4145. void set_pageblock_flags_group(struct page *page, unsigned long flags,
  4146. int start_bitidx, int end_bitidx)
  4147. {
  4148. struct zone *zone;
  4149. unsigned long *bitmap;
  4150. unsigned long pfn, bitidx;
  4151. unsigned long value = 1;
  4152. zone = page_zone(page);
  4153. pfn = page_to_pfn(page);
  4154. bitmap = get_pageblock_bitmap(zone, pfn);
  4155. bitidx = pfn_to_bitidx(zone, pfn);
  4156. VM_BUG_ON(pfn < zone->zone_start_pfn);
  4157. VM_BUG_ON(pfn >= zone->zone_start_pfn + zone->spanned_pages);
  4158. for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1)
  4159. if (flags & value)
  4160. __set_bit(bitidx + start_bitidx, bitmap);
  4161. else
  4162. __clear_bit(bitidx + start_bitidx, bitmap);
  4163. }
  4164. /*
  4165. * This is designed as sub function...plz see page_isolation.c also.
  4166. * set/clear page block's type to be ISOLATE.
  4167. * page allocater never alloc memory from ISOLATE block.
  4168. */
  4169. int set_migratetype_isolate(struct page *page)
  4170. {
  4171. struct zone *zone;
  4172. unsigned long flags;
  4173. int ret = -EBUSY;
  4174. zone = page_zone(page);
  4175. spin_lock_irqsave(&zone->lock, flags);
  4176. /*
  4177. * In future, more migrate types will be able to be isolation target.
  4178. */
  4179. if (get_pageblock_migratetype(page) != MIGRATE_MOVABLE)
  4180. goto out;
  4181. set_pageblock_migratetype(page, MIGRATE_ISOLATE);
  4182. move_freepages_block(zone, page, MIGRATE_ISOLATE);
  4183. ret = 0;
  4184. out:
  4185. spin_unlock_irqrestore(&zone->lock, flags);
  4186. if (!ret)
  4187. drain_all_pages();
  4188. return ret;
  4189. }
  4190. void unset_migratetype_isolate(struct page *page)
  4191. {
  4192. struct zone *zone;
  4193. unsigned long flags;
  4194. zone = page_zone(page);
  4195. spin_lock_irqsave(&zone->lock, flags);
  4196. if (get_pageblock_migratetype(page) != MIGRATE_ISOLATE)
  4197. goto out;
  4198. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  4199. move_freepages_block(zone, page, MIGRATE_MOVABLE);
  4200. out:
  4201. spin_unlock_irqrestore(&zone->lock, flags);
  4202. }
  4203. #ifdef CONFIG_MEMORY_HOTREMOVE
  4204. /*
  4205. * All pages in the range must be isolated before calling this.
  4206. */
  4207. void
  4208. __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
  4209. {
  4210. struct page *page;
  4211. struct zone *zone;
  4212. int order, i;
  4213. unsigned long pfn;
  4214. unsigned long flags;
  4215. /* find the first valid pfn */
  4216. for (pfn = start_pfn; pfn < end_pfn; pfn++)
  4217. if (pfn_valid(pfn))
  4218. break;
  4219. if (pfn == end_pfn)
  4220. return;
  4221. zone = page_zone(pfn_to_page(pfn));
  4222. spin_lock_irqsave(&zone->lock, flags);
  4223. pfn = start_pfn;
  4224. while (pfn < end_pfn) {
  4225. if (!pfn_valid(pfn)) {
  4226. pfn++;
  4227. continue;
  4228. }
  4229. page = pfn_to_page(pfn);
  4230. BUG_ON(page_count(page));
  4231. BUG_ON(!PageBuddy(page));
  4232. order = page_order(page);
  4233. #ifdef CONFIG_DEBUG_VM
  4234. printk(KERN_INFO "remove from free list %lx %d %lx\n",
  4235. pfn, 1 << order, end_pfn);
  4236. #endif
  4237. list_del(&page->lru);
  4238. rmv_page_order(page);
  4239. zone->free_area[order].nr_free--;
  4240. __mod_zone_page_state(zone, NR_FREE_PAGES,
  4241. - (1UL << order));
  4242. for (i = 0; i < (1 << order); i++)
  4243. SetPageReserved((page+i));
  4244. pfn += (1 << order);
  4245. }
  4246. spin_unlock_irqrestore(&zone->lock, flags);
  4247. }
  4248. #endif