page_alloc.c 155 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654
  1. /*
  2. * linux/mm/page_alloc.c
  3. *
  4. * Manages the free list, the system allocates free pages here.
  5. * Note that kmalloc() lives in slab.c
  6. *
  7. * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
  8. * Swap reorganised 29.12.95, Stephen Tweedie
  9. * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
  10. * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
  11. * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
  12. * Zone balancing, Kanoj Sarcar, SGI, Jan 2000
  13. * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
  14. * (lots of bits borrowed from Ingo Molnar & Andrew Morton)
  15. */
  16. #include <linux/stddef.h>
  17. #include <linux/mm.h>
  18. #include <linux/swap.h>
  19. #include <linux/interrupt.h>
  20. #include <linux/pagemap.h>
  21. #include <linux/jiffies.h>
  22. #include <linux/bootmem.h>
  23. #include <linux/memblock.h>
  24. #include <linux/compiler.h>
  25. #include <linux/kernel.h>
  26. #include <linux/kmemcheck.h>
  27. #include <linux/module.h>
  28. #include <linux/suspend.h>
  29. #include <linux/pagevec.h>
  30. #include <linux/blkdev.h>
  31. #include <linux/slab.h>
  32. #include <linux/ratelimit.h>
  33. #include <linux/oom.h>
  34. #include <linux/notifier.h>
  35. #include <linux/topology.h>
  36. #include <linux/sysctl.h>
  37. #include <linux/cpu.h>
  38. #include <linux/cpuset.h>
  39. #include <linux/memory_hotplug.h>
  40. #include <linux/nodemask.h>
  41. #include <linux/vmalloc.h>
  42. #include <linux/vmstat.h>
  43. #include <linux/mempolicy.h>
  44. #include <linux/stop_machine.h>
  45. #include <linux/sort.h>
  46. #include <linux/pfn.h>
  47. #include <linux/backing-dev.h>
  48. #include <linux/fault-inject.h>
  49. #include <linux/page-isolation.h>
  50. #include <linux/page_cgroup.h>
  51. #include <linux/debugobjects.h>
  52. #include <linux/kmemleak.h>
  53. #include <linux/memory.h>
  54. #include <linux/compaction.h>
  55. #include <trace/events/kmem.h>
  56. #include <linux/ftrace_event.h>
  57. #include <linux/memcontrol.h>
  58. #include <linux/prefetch.h>
  59. #include <linux/page-debug-flags.h>
  60. #include <asm/tlbflush.h>
  61. #include <asm/div64.h>
  62. #include "internal.h"
  63. #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
  64. DEFINE_PER_CPU(int, numa_node);
  65. EXPORT_PER_CPU_SYMBOL(numa_node);
  66. #endif
  67. #ifdef CONFIG_HAVE_MEMORYLESS_NODES
  68. /*
  69. * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
  70. * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
  71. * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
  72. * defined in <linux/topology.h>.
  73. */
  74. DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */
  75. EXPORT_PER_CPU_SYMBOL(_numa_mem_);
  76. #endif
  77. /*
  78. * Array of node states.
  79. */
  80. nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
  81. [N_POSSIBLE] = NODE_MASK_ALL,
  82. [N_ONLINE] = { { [0] = 1UL } },
  83. #ifndef CONFIG_NUMA
  84. [N_NORMAL_MEMORY] = { { [0] = 1UL } },
  85. #ifdef CONFIG_HIGHMEM
  86. [N_HIGH_MEMORY] = { { [0] = 1UL } },
  87. #endif
  88. [N_CPU] = { { [0] = 1UL } },
  89. #endif /* NUMA */
  90. };
  91. EXPORT_SYMBOL(node_states);
  92. unsigned long totalram_pages __read_mostly;
  93. unsigned long totalreserve_pages __read_mostly;
  94. /*
  95. * When calculating the number of globally allowed dirty pages, there
  96. * is a certain number of per-zone reserves that should not be
  97. * considered dirtyable memory. This is the sum of those reserves
  98. * over all existing zones that contribute dirtyable memory.
  99. */
  100. unsigned long dirty_balance_reserve __read_mostly;
  101. int percpu_pagelist_fraction;
  102. gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
  103. #ifdef CONFIG_PM_SLEEP
  104. /*
  105. * The following functions are used by the suspend/hibernate code to temporarily
  106. * change gfp_allowed_mask in order to avoid using I/O during memory allocations
  107. * while devices are suspended. To avoid races with the suspend/hibernate code,
  108. * they should always be called with pm_mutex held (gfp_allowed_mask also should
  109. * only be modified with pm_mutex held, unless the suspend/hibernate code is
  110. * guaranteed not to run in parallel with that modification).
  111. */
  112. static gfp_t saved_gfp_mask;
  113. void pm_restore_gfp_mask(void)
  114. {
  115. WARN_ON(!mutex_is_locked(&pm_mutex));
  116. if (saved_gfp_mask) {
  117. gfp_allowed_mask = saved_gfp_mask;
  118. saved_gfp_mask = 0;
  119. }
  120. }
  121. void pm_restrict_gfp_mask(void)
  122. {
  123. WARN_ON(!mutex_is_locked(&pm_mutex));
  124. WARN_ON(saved_gfp_mask);
  125. saved_gfp_mask = gfp_allowed_mask;
  126. gfp_allowed_mask &= ~GFP_IOFS;
  127. }
  128. bool pm_suspended_storage(void)
  129. {
  130. if ((gfp_allowed_mask & GFP_IOFS) == GFP_IOFS)
  131. return false;
  132. return true;
  133. }
  134. #endif /* CONFIG_PM_SLEEP */
  135. #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
  136. int pageblock_order __read_mostly;
  137. #endif
  138. static void __free_pages_ok(struct page *page, unsigned int order);
  139. /*
  140. * results with 256, 32 in the lowmem_reserve sysctl:
  141. * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
  142. * 1G machine -> (16M dma, 784M normal, 224M high)
  143. * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
  144. * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
  145. * HIGHMEM allocation will (224M+784M)/256 of ram reserved in ZONE_DMA
  146. *
  147. * TBD: should special case ZONE_DMA32 machines here - in those we normally
  148. * don't need any ZONE_NORMAL reservation
  149. */
  150. int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = {
  151. #ifdef CONFIG_ZONE_DMA
  152. 256,
  153. #endif
  154. #ifdef CONFIG_ZONE_DMA32
  155. 256,
  156. #endif
  157. #ifdef CONFIG_HIGHMEM
  158. 32,
  159. #endif
  160. 32,
  161. };
  162. EXPORT_SYMBOL(totalram_pages);
  163. static char * const zone_names[MAX_NR_ZONES] = {
  164. #ifdef CONFIG_ZONE_DMA
  165. "DMA",
  166. #endif
  167. #ifdef CONFIG_ZONE_DMA32
  168. "DMA32",
  169. #endif
  170. "Normal",
  171. #ifdef CONFIG_HIGHMEM
  172. "HighMem",
  173. #endif
  174. "Movable",
  175. };
  176. int min_free_kbytes = 1024;
  177. static unsigned long __meminitdata nr_kernel_pages;
  178. static unsigned long __meminitdata nr_all_pages;
  179. static unsigned long __meminitdata dma_reserve;
  180. #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
  181. static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES];
  182. static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES];
  183. static unsigned long __initdata required_kernelcore;
  184. static unsigned long __initdata required_movablecore;
  185. static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES];
  186. /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
  187. int movable_zone;
  188. EXPORT_SYMBOL(movable_zone);
  189. #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
  190. #if MAX_NUMNODES > 1
  191. int nr_node_ids __read_mostly = MAX_NUMNODES;
  192. int nr_online_nodes __read_mostly = 1;
  193. EXPORT_SYMBOL(nr_node_ids);
  194. EXPORT_SYMBOL(nr_online_nodes);
  195. #endif
  196. int page_group_by_mobility_disabled __read_mostly;
  197. static void set_pageblock_migratetype(struct page *page, int migratetype)
  198. {
  199. if (unlikely(page_group_by_mobility_disabled))
  200. migratetype = MIGRATE_UNMOVABLE;
  201. set_pageblock_flags_group(page, (unsigned long)migratetype,
  202. PB_migrate, PB_migrate_end);
  203. }
  204. bool oom_killer_disabled __read_mostly;
  205. #ifdef CONFIG_DEBUG_VM
  206. static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
  207. {
  208. int ret = 0;
  209. unsigned seq;
  210. unsigned long pfn = page_to_pfn(page);
  211. do {
  212. seq = zone_span_seqbegin(zone);
  213. if (pfn >= zone->zone_start_pfn + zone->spanned_pages)
  214. ret = 1;
  215. else if (pfn < zone->zone_start_pfn)
  216. ret = 1;
  217. } while (zone_span_seqretry(zone, seq));
  218. return ret;
  219. }
  220. static int page_is_consistent(struct zone *zone, struct page *page)
  221. {
  222. if (!pfn_valid_within(page_to_pfn(page)))
  223. return 0;
  224. if (zone != page_zone(page))
  225. return 0;
  226. return 1;
  227. }
  228. /*
  229. * Temporary debugging check for pages not lying within a given zone.
  230. */
  231. static int bad_range(struct zone *zone, struct page *page)
  232. {
  233. if (page_outside_zone_boundaries(zone, page))
  234. return 1;
  235. if (!page_is_consistent(zone, page))
  236. return 1;
  237. return 0;
  238. }
  239. #else
  240. static inline int bad_range(struct zone *zone, struct page *page)
  241. {
  242. return 0;
  243. }
  244. #endif
  245. static void bad_page(struct page *page)
  246. {
  247. static unsigned long resume;
  248. static unsigned long nr_shown;
  249. static unsigned long nr_unshown;
  250. /* Don't complain about poisoned pages */
  251. if (PageHWPoison(page)) {
  252. reset_page_mapcount(page); /* remove PageBuddy */
  253. return;
  254. }
  255. /*
  256. * Allow a burst of 60 reports, then keep quiet for that minute;
  257. * or allow a steady drip of one report per second.
  258. */
  259. if (nr_shown == 60) {
  260. if (time_before(jiffies, resume)) {
  261. nr_unshown++;
  262. goto out;
  263. }
  264. if (nr_unshown) {
  265. printk(KERN_ALERT
  266. "BUG: Bad page state: %lu messages suppressed\n",
  267. nr_unshown);
  268. nr_unshown = 0;
  269. }
  270. nr_shown = 0;
  271. }
  272. if (nr_shown++ == 0)
  273. resume = jiffies + 60 * HZ;
  274. printk(KERN_ALERT "BUG: Bad page state in process %s pfn:%05lx\n",
  275. current->comm, page_to_pfn(page));
  276. dump_page(page);
  277. print_modules();
  278. dump_stack();
  279. out:
  280. /* Leave bad fields for debug, except PageBuddy could make trouble */
  281. reset_page_mapcount(page); /* remove PageBuddy */
  282. add_taint(TAINT_BAD_PAGE);
  283. }
  284. /*
  285. * Higher-order pages are called "compound pages". They are structured thusly:
  286. *
  287. * The first PAGE_SIZE page is called the "head page".
  288. *
  289. * The remaining PAGE_SIZE pages are called "tail pages".
  290. *
  291. * All pages have PG_compound set. All tail pages have their ->first_page
  292. * pointing at the head page.
  293. *
  294. * The first tail page's ->lru.next holds the address of the compound page's
  295. * put_page() function. Its ->lru.prev holds the order of allocation.
  296. * This usage means that zero-order pages may not be compound.
  297. */
  298. static void free_compound_page(struct page *page)
  299. {
  300. __free_pages_ok(page, compound_order(page));
  301. }
  302. void prep_compound_page(struct page *page, unsigned long order)
  303. {
  304. int i;
  305. int nr_pages = 1 << order;
  306. set_compound_page_dtor(page, free_compound_page);
  307. set_compound_order(page, order);
  308. __SetPageHead(page);
  309. for (i = 1; i < nr_pages; i++) {
  310. struct page *p = page + i;
  311. __SetPageTail(p);
  312. set_page_count(p, 0);
  313. p->first_page = page;
  314. }
  315. }
  316. /* update __split_huge_page_refcount if you change this function */
  317. static int destroy_compound_page(struct page *page, unsigned long order)
  318. {
  319. int i;
  320. int nr_pages = 1 << order;
  321. int bad = 0;
  322. if (unlikely(compound_order(page) != order) ||
  323. unlikely(!PageHead(page))) {
  324. bad_page(page);
  325. bad++;
  326. }
  327. __ClearPageHead(page);
  328. for (i = 1; i < nr_pages; i++) {
  329. struct page *p = page + i;
  330. if (unlikely(!PageTail(p) || (p->first_page != page))) {
  331. bad_page(page);
  332. bad++;
  333. }
  334. __ClearPageTail(p);
  335. }
  336. return bad;
  337. }
  338. static inline void prep_zero_page(struct page *page, int order, gfp_t gfp_flags)
  339. {
  340. int i;
  341. /*
  342. * clear_highpage() will use KM_USER0, so it's a bug to use __GFP_ZERO
  343. * and __GFP_HIGHMEM from hard or soft interrupt context.
  344. */
  345. VM_BUG_ON((gfp_flags & __GFP_HIGHMEM) && in_interrupt());
  346. for (i = 0; i < (1 << order); i++)
  347. clear_highpage(page + i);
  348. }
  349. #ifdef CONFIG_DEBUG_PAGEALLOC
  350. unsigned int _debug_guardpage_minorder;
  351. static int __init debug_guardpage_minorder_setup(char *buf)
  352. {
  353. unsigned long res;
  354. if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) {
  355. printk(KERN_ERR "Bad debug_guardpage_minorder value\n");
  356. return 0;
  357. }
  358. _debug_guardpage_minorder = res;
  359. printk(KERN_INFO "Setting debug_guardpage_minorder to %lu\n", res);
  360. return 0;
  361. }
  362. __setup("debug_guardpage_minorder=", debug_guardpage_minorder_setup);
  363. static inline void set_page_guard_flag(struct page *page)
  364. {
  365. __set_bit(PAGE_DEBUG_FLAG_GUARD, &page->debug_flags);
  366. }
  367. static inline void clear_page_guard_flag(struct page *page)
  368. {
  369. __clear_bit(PAGE_DEBUG_FLAG_GUARD, &page->debug_flags);
  370. }
  371. #else
  372. static inline void set_page_guard_flag(struct page *page) { }
  373. static inline void clear_page_guard_flag(struct page *page) { }
  374. #endif
  375. static inline void set_page_order(struct page *page, int order)
  376. {
  377. set_page_private(page, order);
  378. __SetPageBuddy(page);
  379. }
  380. static inline void rmv_page_order(struct page *page)
  381. {
  382. __ClearPageBuddy(page);
  383. set_page_private(page, 0);
  384. }
  385. /*
  386. * Locate the struct page for both the matching buddy in our
  387. * pair (buddy1) and the combined O(n+1) page they form (page).
  388. *
  389. * 1) Any buddy B1 will have an order O twin B2 which satisfies
  390. * the following equation:
  391. * B2 = B1 ^ (1 << O)
  392. * For example, if the starting buddy (buddy2) is #8 its order
  393. * 1 buddy is #10:
  394. * B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10
  395. *
  396. * 2) Any buddy B will have an order O+1 parent P which
  397. * satisfies the following equation:
  398. * P = B & ~(1 << O)
  399. *
  400. * Assumption: *_mem_map is contiguous at least up to MAX_ORDER
  401. */
  402. static inline unsigned long
  403. __find_buddy_index(unsigned long page_idx, unsigned int order)
  404. {
  405. return page_idx ^ (1 << order);
  406. }
  407. /*
  408. * This function checks whether a page is free && is the buddy
  409. * we can do coalesce a page and its buddy if
  410. * (a) the buddy is not in a hole &&
  411. * (b) the buddy is in the buddy system &&
  412. * (c) a page and its buddy have the same order &&
  413. * (d) a page and its buddy are in the same zone.
  414. *
  415. * For recording whether a page is in the buddy system, we set ->_mapcount -2.
  416. * Setting, clearing, and testing _mapcount -2 is serialized by zone->lock.
  417. *
  418. * For recording page's order, we use page_private(page).
  419. */
  420. static inline int page_is_buddy(struct page *page, struct page *buddy,
  421. int order)
  422. {
  423. if (!pfn_valid_within(page_to_pfn(buddy)))
  424. return 0;
  425. if (page_zone_id(page) != page_zone_id(buddy))
  426. return 0;
  427. if (page_is_guard(buddy) && page_order(buddy) == order) {
  428. VM_BUG_ON(page_count(buddy) != 0);
  429. return 1;
  430. }
  431. if (PageBuddy(buddy) && page_order(buddy) == order) {
  432. VM_BUG_ON(page_count(buddy) != 0);
  433. return 1;
  434. }
  435. return 0;
  436. }
  437. /*
  438. * Freeing function for a buddy system allocator.
  439. *
  440. * The concept of a buddy system is to maintain direct-mapped table
  441. * (containing bit values) for memory blocks of various "orders".
  442. * The bottom level table contains the map for the smallest allocatable
  443. * units of memory (here, pages), and each level above it describes
  444. * pairs of units from the levels below, hence, "buddies".
  445. * At a high level, all that happens here is marking the table entry
  446. * at the bottom level available, and propagating the changes upward
  447. * as necessary, plus some accounting needed to play nicely with other
  448. * parts of the VM system.
  449. * At each level, we keep a list of pages, which are heads of continuous
  450. * free pages of length of (1 << order) and marked with _mapcount -2. Page's
  451. * order is recorded in page_private(page) field.
  452. * So when we are allocating or freeing one, we can derive the state of the
  453. * other. That is, if we allocate a small block, and both were
  454. * free, the remainder of the region must be split into blocks.
  455. * If a block is freed, and its buddy is also free, then this
  456. * triggers coalescing into a block of larger size.
  457. *
  458. * -- wli
  459. */
  460. static inline void __free_one_page(struct page *page,
  461. struct zone *zone, unsigned int order,
  462. int migratetype)
  463. {
  464. unsigned long page_idx;
  465. unsigned long combined_idx;
  466. unsigned long uninitialized_var(buddy_idx);
  467. struct page *buddy;
  468. if (unlikely(PageCompound(page)))
  469. if (unlikely(destroy_compound_page(page, order)))
  470. return;
  471. VM_BUG_ON(migratetype == -1);
  472. page_idx = page_to_pfn(page) & ((1 << MAX_ORDER) - 1);
  473. VM_BUG_ON(page_idx & ((1 << order) - 1));
  474. VM_BUG_ON(bad_range(zone, page));
  475. while (order < MAX_ORDER-1) {
  476. buddy_idx = __find_buddy_index(page_idx, order);
  477. buddy = page + (buddy_idx - page_idx);
  478. if (!page_is_buddy(page, buddy, order))
  479. break;
  480. /*
  481. * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
  482. * merge with it and move up one order.
  483. */
  484. if (page_is_guard(buddy)) {
  485. clear_page_guard_flag(buddy);
  486. set_page_private(page, 0);
  487. __mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order);
  488. } else {
  489. list_del(&buddy->lru);
  490. zone->free_area[order].nr_free--;
  491. rmv_page_order(buddy);
  492. }
  493. combined_idx = buddy_idx & page_idx;
  494. page = page + (combined_idx - page_idx);
  495. page_idx = combined_idx;
  496. order++;
  497. }
  498. set_page_order(page, order);
  499. /*
  500. * If this is not the largest possible page, check if the buddy
  501. * of the next-highest order is free. If it is, it's possible
  502. * that pages are being freed that will coalesce soon. In case,
  503. * that is happening, add the free page to the tail of the list
  504. * so it's less likely to be used soon and more likely to be merged
  505. * as a higher order page
  506. */
  507. if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {
  508. struct page *higher_page, *higher_buddy;
  509. combined_idx = buddy_idx & page_idx;
  510. higher_page = page + (combined_idx - page_idx);
  511. buddy_idx = __find_buddy_index(combined_idx, order + 1);
  512. higher_buddy = page + (buddy_idx - combined_idx);
  513. if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
  514. list_add_tail(&page->lru,
  515. &zone->free_area[order].free_list[migratetype]);
  516. goto out;
  517. }
  518. }
  519. list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
  520. out:
  521. zone->free_area[order].nr_free++;
  522. }
  523. /*
  524. * free_page_mlock() -- clean up attempts to free and mlocked() page.
  525. * Page should not be on lru, so no need to fix that up.
  526. * free_pages_check() will verify...
  527. */
  528. static inline void free_page_mlock(struct page *page)
  529. {
  530. __dec_zone_page_state(page, NR_MLOCK);
  531. __count_vm_event(UNEVICTABLE_MLOCKFREED);
  532. }
  533. static inline int free_pages_check(struct page *page)
  534. {
  535. if (unlikely(page_mapcount(page) |
  536. (page->mapping != NULL) |
  537. (atomic_read(&page->_count) != 0) |
  538. (page->flags & PAGE_FLAGS_CHECK_AT_FREE) |
  539. (mem_cgroup_bad_page_check(page)))) {
  540. bad_page(page);
  541. return 1;
  542. }
  543. if (page->flags & PAGE_FLAGS_CHECK_AT_PREP)
  544. page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
  545. return 0;
  546. }
  547. /*
  548. * Frees a number of pages from the PCP lists
  549. * Assumes all pages on list are in same zone, and of same order.
  550. * count is the number of pages to free.
  551. *
  552. * If the zone was previously in an "all pages pinned" state then look to
  553. * see if this freeing clears that state.
  554. *
  555. * And clear the zone's pages_scanned counter, to hold off the "all pages are
  556. * pinned" detection logic.
  557. */
  558. static void free_pcppages_bulk(struct zone *zone, int count,
  559. struct per_cpu_pages *pcp)
  560. {
  561. int migratetype = 0;
  562. int batch_free = 0;
  563. int to_free = count;
  564. spin_lock(&zone->lock);
  565. zone->all_unreclaimable = 0;
  566. zone->pages_scanned = 0;
  567. while (to_free) {
  568. struct page *page;
  569. struct list_head *list;
  570. /*
  571. * Remove pages from lists in a round-robin fashion. A
  572. * batch_free count is maintained that is incremented when an
  573. * empty list is encountered. This is so more pages are freed
  574. * off fuller lists instead of spinning excessively around empty
  575. * lists
  576. */
  577. do {
  578. batch_free++;
  579. if (++migratetype == MIGRATE_PCPTYPES)
  580. migratetype = 0;
  581. list = &pcp->lists[migratetype];
  582. } while (list_empty(list));
  583. /* This is the only non-empty list. Free them all. */
  584. if (batch_free == MIGRATE_PCPTYPES)
  585. batch_free = to_free;
  586. do {
  587. page = list_entry(list->prev, struct page, lru);
  588. /* must delete as __free_one_page list manipulates */
  589. list_del(&page->lru);
  590. /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */
  591. __free_one_page(page, zone, 0, page_private(page));
  592. trace_mm_page_pcpu_drain(page, 0, page_private(page));
  593. } while (--to_free && --batch_free && !list_empty(list));
  594. }
  595. __mod_zone_page_state(zone, NR_FREE_PAGES, count);
  596. spin_unlock(&zone->lock);
  597. }
  598. static void free_one_page(struct zone *zone, struct page *page, int order,
  599. int migratetype)
  600. {
  601. spin_lock(&zone->lock);
  602. zone->all_unreclaimable = 0;
  603. zone->pages_scanned = 0;
  604. __free_one_page(page, zone, order, migratetype);
  605. __mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order);
  606. spin_unlock(&zone->lock);
  607. }
  608. static bool free_pages_prepare(struct page *page, unsigned int order)
  609. {
  610. int i;
  611. int bad = 0;
  612. trace_mm_page_free(page, order);
  613. kmemcheck_free_shadow(page, order);
  614. if (PageAnon(page))
  615. page->mapping = NULL;
  616. for (i = 0; i < (1 << order); i++)
  617. bad += free_pages_check(page + i);
  618. if (bad)
  619. return false;
  620. if (!PageHighMem(page)) {
  621. debug_check_no_locks_freed(page_address(page),PAGE_SIZE<<order);
  622. debug_check_no_obj_freed(page_address(page),
  623. PAGE_SIZE << order);
  624. }
  625. arch_free_page(page, order);
  626. kernel_map_pages(page, 1 << order, 0);
  627. return true;
  628. }
  629. static void __free_pages_ok(struct page *page, unsigned int order)
  630. {
  631. unsigned long flags;
  632. int wasMlocked = __TestClearPageMlocked(page);
  633. if (!free_pages_prepare(page, order))
  634. return;
  635. local_irq_save(flags);
  636. if (unlikely(wasMlocked))
  637. free_page_mlock(page);
  638. __count_vm_events(PGFREE, 1 << order);
  639. free_one_page(page_zone(page), page, order,
  640. get_pageblock_migratetype(page));
  641. local_irq_restore(flags);
  642. }
  643. void __meminit __free_pages_bootmem(struct page *page, unsigned int order)
  644. {
  645. unsigned int nr_pages = 1 << order;
  646. unsigned int loop;
  647. prefetchw(page);
  648. for (loop = 0; loop < nr_pages; loop++) {
  649. struct page *p = &page[loop];
  650. if (loop + 1 < nr_pages)
  651. prefetchw(p + 1);
  652. __ClearPageReserved(p);
  653. set_page_count(p, 0);
  654. }
  655. set_page_refcounted(page);
  656. __free_pages(page, order);
  657. }
  658. /*
  659. * The order of subdivision here is critical for the IO subsystem.
  660. * Please do not alter this order without good reasons and regression
  661. * testing. Specifically, as large blocks of memory are subdivided,
  662. * the order in which smaller blocks are delivered depends on the order
  663. * they're subdivided in this function. This is the primary factor
  664. * influencing the order in which pages are delivered to the IO
  665. * subsystem according to empirical testing, and this is also justified
  666. * by considering the behavior of a buddy system containing a single
  667. * large block of memory acted on by a series of small allocations.
  668. * This behavior is a critical factor in sglist merging's success.
  669. *
  670. * -- wli
  671. */
  672. static inline void expand(struct zone *zone, struct page *page,
  673. int low, int high, struct free_area *area,
  674. int migratetype)
  675. {
  676. unsigned long size = 1 << high;
  677. while (high > low) {
  678. area--;
  679. high--;
  680. size >>= 1;
  681. VM_BUG_ON(bad_range(zone, &page[size]));
  682. #ifdef CONFIG_DEBUG_PAGEALLOC
  683. if (high < debug_guardpage_minorder()) {
  684. /*
  685. * Mark as guard pages (or page), that will allow to
  686. * merge back to allocator when buddy will be freed.
  687. * Corresponding page table entries will not be touched,
  688. * pages will stay not present in virtual address space
  689. */
  690. INIT_LIST_HEAD(&page[size].lru);
  691. set_page_guard_flag(&page[size]);
  692. set_page_private(&page[size], high);
  693. /* Guard pages are not available for any usage */
  694. __mod_zone_page_state(zone, NR_FREE_PAGES, -(1 << high));
  695. continue;
  696. }
  697. #endif
  698. list_add(&page[size].lru, &area->free_list[migratetype]);
  699. area->nr_free++;
  700. set_page_order(&page[size], high);
  701. }
  702. }
  703. /*
  704. * This page is about to be returned from the page allocator
  705. */
  706. static inline int check_new_page(struct page *page)
  707. {
  708. if (unlikely(page_mapcount(page) |
  709. (page->mapping != NULL) |
  710. (atomic_read(&page->_count) != 0) |
  711. (page->flags & PAGE_FLAGS_CHECK_AT_PREP) |
  712. (mem_cgroup_bad_page_check(page)))) {
  713. bad_page(page);
  714. return 1;
  715. }
  716. return 0;
  717. }
  718. static int prep_new_page(struct page *page, int order, gfp_t gfp_flags)
  719. {
  720. int i;
  721. for (i = 0; i < (1 << order); i++) {
  722. struct page *p = page + i;
  723. if (unlikely(check_new_page(p)))
  724. return 1;
  725. }
  726. set_page_private(page, 0);
  727. set_page_refcounted(page);
  728. arch_alloc_page(page, order);
  729. kernel_map_pages(page, 1 << order, 1);
  730. if (gfp_flags & __GFP_ZERO)
  731. prep_zero_page(page, order, gfp_flags);
  732. if (order && (gfp_flags & __GFP_COMP))
  733. prep_compound_page(page, order);
  734. return 0;
  735. }
  736. /*
  737. * Go through the free lists for the given migratetype and remove
  738. * the smallest available page from the freelists
  739. */
  740. static inline
  741. struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
  742. int migratetype)
  743. {
  744. unsigned int current_order;
  745. struct free_area * area;
  746. struct page *page;
  747. /* Find a page of the appropriate size in the preferred list */
  748. for (current_order = order; current_order < MAX_ORDER; ++current_order) {
  749. area = &(zone->free_area[current_order]);
  750. if (list_empty(&area->free_list[migratetype]))
  751. continue;
  752. page = list_entry(area->free_list[migratetype].next,
  753. struct page, lru);
  754. list_del(&page->lru);
  755. rmv_page_order(page);
  756. area->nr_free--;
  757. expand(zone, page, order, current_order, area, migratetype);
  758. return page;
  759. }
  760. return NULL;
  761. }
  762. /*
  763. * This array describes the order lists are fallen back to when
  764. * the free lists for the desirable migrate type are depleted
  765. */
  766. static int fallbacks[MIGRATE_TYPES][MIGRATE_TYPES-1] = {
  767. [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE },
  768. [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE },
  769. [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },
  770. [MIGRATE_RESERVE] = { MIGRATE_RESERVE, MIGRATE_RESERVE, MIGRATE_RESERVE }, /* Never used */
  771. };
  772. /*
  773. * Move the free pages in a range to the free lists of the requested type.
  774. * Note that start_page and end_pages are not aligned on a pageblock
  775. * boundary. If alignment is required, use move_freepages_block()
  776. */
  777. static int move_freepages(struct zone *zone,
  778. struct page *start_page, struct page *end_page,
  779. int migratetype)
  780. {
  781. struct page *page;
  782. unsigned long order;
  783. int pages_moved = 0;
  784. #ifndef CONFIG_HOLES_IN_ZONE
  785. /*
  786. * page_zone is not safe to call in this context when
  787. * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant
  788. * anyway as we check zone boundaries in move_freepages_block().
  789. * Remove at a later date when no bug reports exist related to
  790. * grouping pages by mobility
  791. */
  792. BUG_ON(page_zone(start_page) != page_zone(end_page));
  793. #endif
  794. for (page = start_page; page <= end_page;) {
  795. /* Make sure we are not inadvertently changing nodes */
  796. VM_BUG_ON(page_to_nid(page) != zone_to_nid(zone));
  797. if (!pfn_valid_within(page_to_pfn(page))) {
  798. page++;
  799. continue;
  800. }
  801. if (!PageBuddy(page)) {
  802. page++;
  803. continue;
  804. }
  805. order = page_order(page);
  806. list_move(&page->lru,
  807. &zone->free_area[order].free_list[migratetype]);
  808. page += 1 << order;
  809. pages_moved += 1 << order;
  810. }
  811. return pages_moved;
  812. }
  813. static int move_freepages_block(struct zone *zone, struct page *page,
  814. int migratetype)
  815. {
  816. unsigned long start_pfn, end_pfn;
  817. struct page *start_page, *end_page;
  818. start_pfn = page_to_pfn(page);
  819. start_pfn = start_pfn & ~(pageblock_nr_pages-1);
  820. start_page = pfn_to_page(start_pfn);
  821. end_page = start_page + pageblock_nr_pages - 1;
  822. end_pfn = start_pfn + pageblock_nr_pages - 1;
  823. /* Do not cross zone boundaries */
  824. if (start_pfn < zone->zone_start_pfn)
  825. start_page = page;
  826. if (end_pfn >= zone->zone_start_pfn + zone->spanned_pages)
  827. return 0;
  828. return move_freepages(zone, start_page, end_page, migratetype);
  829. }
  830. static void change_pageblock_range(struct page *pageblock_page,
  831. int start_order, int migratetype)
  832. {
  833. int nr_pageblocks = 1 << (start_order - pageblock_order);
  834. while (nr_pageblocks--) {
  835. set_pageblock_migratetype(pageblock_page, migratetype);
  836. pageblock_page += pageblock_nr_pages;
  837. }
  838. }
  839. /* Remove an element from the buddy allocator from the fallback list */
  840. static inline struct page *
  841. __rmqueue_fallback(struct zone *zone, int order, int start_migratetype)
  842. {
  843. struct free_area * area;
  844. int current_order;
  845. struct page *page;
  846. int migratetype, i;
  847. /* Find the largest possible block of pages in the other list */
  848. for (current_order = MAX_ORDER-1; current_order >= order;
  849. --current_order) {
  850. for (i = 0; i < MIGRATE_TYPES - 1; i++) {
  851. migratetype = fallbacks[start_migratetype][i];
  852. /* MIGRATE_RESERVE handled later if necessary */
  853. if (migratetype == MIGRATE_RESERVE)
  854. continue;
  855. area = &(zone->free_area[current_order]);
  856. if (list_empty(&area->free_list[migratetype]))
  857. continue;
  858. page = list_entry(area->free_list[migratetype].next,
  859. struct page, lru);
  860. area->nr_free--;
  861. /*
  862. * If breaking a large block of pages, move all free
  863. * pages to the preferred allocation list. If falling
  864. * back for a reclaimable kernel allocation, be more
  865. * aggressive about taking ownership of free pages
  866. */
  867. if (unlikely(current_order >= (pageblock_order >> 1)) ||
  868. start_migratetype == MIGRATE_RECLAIMABLE ||
  869. page_group_by_mobility_disabled) {
  870. unsigned long pages;
  871. pages = move_freepages_block(zone, page,
  872. start_migratetype);
  873. /* Claim the whole block if over half of it is free */
  874. if (pages >= (1 << (pageblock_order-1)) ||
  875. page_group_by_mobility_disabled)
  876. set_pageblock_migratetype(page,
  877. start_migratetype);
  878. migratetype = start_migratetype;
  879. }
  880. /* Remove the page from the freelists */
  881. list_del(&page->lru);
  882. rmv_page_order(page);
  883. /* Take ownership for orders >= pageblock_order */
  884. if (current_order >= pageblock_order)
  885. change_pageblock_range(page, current_order,
  886. start_migratetype);
  887. expand(zone, page, order, current_order, area, migratetype);
  888. trace_mm_page_alloc_extfrag(page, order, current_order,
  889. start_migratetype, migratetype);
  890. return page;
  891. }
  892. }
  893. return NULL;
  894. }
  895. /*
  896. * Do the hard work of removing an element from the buddy allocator.
  897. * Call me with the zone->lock already held.
  898. */
  899. static struct page *__rmqueue(struct zone *zone, unsigned int order,
  900. int migratetype)
  901. {
  902. struct page *page;
  903. retry_reserve:
  904. page = __rmqueue_smallest(zone, order, migratetype);
  905. if (unlikely(!page) && migratetype != MIGRATE_RESERVE) {
  906. page = __rmqueue_fallback(zone, order, migratetype);
  907. /*
  908. * Use MIGRATE_RESERVE rather than fail an allocation. goto
  909. * is used because __rmqueue_smallest is an inline function
  910. * and we want just one call site
  911. */
  912. if (!page) {
  913. migratetype = MIGRATE_RESERVE;
  914. goto retry_reserve;
  915. }
  916. }
  917. trace_mm_page_alloc_zone_locked(page, order, migratetype);
  918. return page;
  919. }
  920. /*
  921. * Obtain a specified number of elements from the buddy allocator, all under
  922. * a single hold of the lock, for efficiency. Add them to the supplied list.
  923. * Returns the number of new pages which were placed at *list.
  924. */
  925. static int rmqueue_bulk(struct zone *zone, unsigned int order,
  926. unsigned long count, struct list_head *list,
  927. int migratetype, int cold)
  928. {
  929. int i;
  930. spin_lock(&zone->lock);
  931. for (i = 0; i < count; ++i) {
  932. struct page *page = __rmqueue(zone, order, migratetype);
  933. if (unlikely(page == NULL))
  934. break;
  935. /*
  936. * Split buddy pages returned by expand() are received here
  937. * in physical page order. The page is added to the callers and
  938. * list and the list head then moves forward. From the callers
  939. * perspective, the linked list is ordered by page number in
  940. * some conditions. This is useful for IO devices that can
  941. * merge IO requests if the physical pages are ordered
  942. * properly.
  943. */
  944. if (likely(cold == 0))
  945. list_add(&page->lru, list);
  946. else
  947. list_add_tail(&page->lru, list);
  948. set_page_private(page, migratetype);
  949. list = &page->lru;
  950. }
  951. __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
  952. spin_unlock(&zone->lock);
  953. return i;
  954. }
  955. #ifdef CONFIG_NUMA
  956. /*
  957. * Called from the vmstat counter updater to drain pagesets of this
  958. * currently executing processor on remote nodes after they have
  959. * expired.
  960. *
  961. * Note that this function must be called with the thread pinned to
  962. * a single processor.
  963. */
  964. void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
  965. {
  966. unsigned long flags;
  967. int to_drain;
  968. local_irq_save(flags);
  969. if (pcp->count >= pcp->batch)
  970. to_drain = pcp->batch;
  971. else
  972. to_drain = pcp->count;
  973. free_pcppages_bulk(zone, to_drain, pcp);
  974. pcp->count -= to_drain;
  975. local_irq_restore(flags);
  976. }
  977. #endif
  978. /*
  979. * Drain pages of the indicated processor.
  980. *
  981. * The processor must either be the current processor and the
  982. * thread pinned to the current processor or a processor that
  983. * is not online.
  984. */
  985. static void drain_pages(unsigned int cpu)
  986. {
  987. unsigned long flags;
  988. struct zone *zone;
  989. for_each_populated_zone(zone) {
  990. struct per_cpu_pageset *pset;
  991. struct per_cpu_pages *pcp;
  992. local_irq_save(flags);
  993. pset = per_cpu_ptr(zone->pageset, cpu);
  994. pcp = &pset->pcp;
  995. if (pcp->count) {
  996. free_pcppages_bulk(zone, pcp->count, pcp);
  997. pcp->count = 0;
  998. }
  999. local_irq_restore(flags);
  1000. }
  1001. }
  1002. /*
  1003. * Spill all of this CPU's per-cpu pages back into the buddy allocator.
  1004. */
  1005. void drain_local_pages(void *arg)
  1006. {
  1007. drain_pages(smp_processor_id());
  1008. }
  1009. /*
  1010. * Spill all the per-cpu pages from all CPUs back into the buddy allocator
  1011. */
  1012. void drain_all_pages(void)
  1013. {
  1014. on_each_cpu(drain_local_pages, NULL, 1);
  1015. }
  1016. #ifdef CONFIG_HIBERNATION
  1017. void mark_free_pages(struct zone *zone)
  1018. {
  1019. unsigned long pfn, max_zone_pfn;
  1020. unsigned long flags;
  1021. int order, t;
  1022. struct list_head *curr;
  1023. if (!zone->spanned_pages)
  1024. return;
  1025. spin_lock_irqsave(&zone->lock, flags);
  1026. max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
  1027. for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
  1028. if (pfn_valid(pfn)) {
  1029. struct page *page = pfn_to_page(pfn);
  1030. if (!swsusp_page_is_forbidden(page))
  1031. swsusp_unset_page_free(page);
  1032. }
  1033. for_each_migratetype_order(order, t) {
  1034. list_for_each(curr, &zone->free_area[order].free_list[t]) {
  1035. unsigned long i;
  1036. pfn = page_to_pfn(list_entry(curr, struct page, lru));
  1037. for (i = 0; i < (1UL << order); i++)
  1038. swsusp_set_page_free(pfn_to_page(pfn + i));
  1039. }
  1040. }
  1041. spin_unlock_irqrestore(&zone->lock, flags);
  1042. }
  1043. #endif /* CONFIG_PM */
  1044. /*
  1045. * Free a 0-order page
  1046. * cold == 1 ? free a cold page : free a hot page
  1047. */
  1048. void free_hot_cold_page(struct page *page, int cold)
  1049. {
  1050. struct zone *zone = page_zone(page);
  1051. struct per_cpu_pages *pcp;
  1052. unsigned long flags;
  1053. int migratetype;
  1054. int wasMlocked = __TestClearPageMlocked(page);
  1055. if (!free_pages_prepare(page, 0))
  1056. return;
  1057. migratetype = get_pageblock_migratetype(page);
  1058. set_page_private(page, migratetype);
  1059. local_irq_save(flags);
  1060. if (unlikely(wasMlocked))
  1061. free_page_mlock(page);
  1062. __count_vm_event(PGFREE);
  1063. /*
  1064. * We only track unmovable, reclaimable and movable on pcp lists.
  1065. * Free ISOLATE pages back to the allocator because they are being
  1066. * offlined but treat RESERVE as movable pages so we can get those
  1067. * areas back if necessary. Otherwise, we may have to free
  1068. * excessively into the page allocator
  1069. */
  1070. if (migratetype >= MIGRATE_PCPTYPES) {
  1071. if (unlikely(migratetype == MIGRATE_ISOLATE)) {
  1072. free_one_page(zone, page, 0, migratetype);
  1073. goto out;
  1074. }
  1075. migratetype = MIGRATE_MOVABLE;
  1076. }
  1077. pcp = &this_cpu_ptr(zone->pageset)->pcp;
  1078. if (cold)
  1079. list_add_tail(&page->lru, &pcp->lists[migratetype]);
  1080. else
  1081. list_add(&page->lru, &pcp->lists[migratetype]);
  1082. pcp->count++;
  1083. if (pcp->count >= pcp->high) {
  1084. free_pcppages_bulk(zone, pcp->batch, pcp);
  1085. pcp->count -= pcp->batch;
  1086. }
  1087. out:
  1088. local_irq_restore(flags);
  1089. }
  1090. /*
  1091. * Free a list of 0-order pages
  1092. */
  1093. void free_hot_cold_page_list(struct list_head *list, int cold)
  1094. {
  1095. struct page *page, *next;
  1096. list_for_each_entry_safe(page, next, list, lru) {
  1097. trace_mm_page_free_batched(page, cold);
  1098. free_hot_cold_page(page, cold);
  1099. }
  1100. }
  1101. /*
  1102. * split_page takes a non-compound higher-order page, and splits it into
  1103. * n (1<<order) sub-pages: page[0..n]
  1104. * Each sub-page must be freed individually.
  1105. *
  1106. * Note: this is probably too low level an operation for use in drivers.
  1107. * Please consult with lkml before using this in your driver.
  1108. */
  1109. void split_page(struct page *page, unsigned int order)
  1110. {
  1111. int i;
  1112. VM_BUG_ON(PageCompound(page));
  1113. VM_BUG_ON(!page_count(page));
  1114. #ifdef CONFIG_KMEMCHECK
  1115. /*
  1116. * Split shadow pages too, because free(page[0]) would
  1117. * otherwise free the whole shadow.
  1118. */
  1119. if (kmemcheck_page_is_tracked(page))
  1120. split_page(virt_to_page(page[0].shadow), order);
  1121. #endif
  1122. for (i = 1; i < (1 << order); i++)
  1123. set_page_refcounted(page + i);
  1124. }
  1125. /*
  1126. * Similar to split_page except the page is already free. As this is only
  1127. * being used for migration, the migratetype of the block also changes.
  1128. * As this is called with interrupts disabled, the caller is responsible
  1129. * for calling arch_alloc_page() and kernel_map_page() after interrupts
  1130. * are enabled.
  1131. *
  1132. * Note: this is probably too low level an operation for use in drivers.
  1133. * Please consult with lkml before using this in your driver.
  1134. */
  1135. int split_free_page(struct page *page)
  1136. {
  1137. unsigned int order;
  1138. unsigned long watermark;
  1139. struct zone *zone;
  1140. BUG_ON(!PageBuddy(page));
  1141. zone = page_zone(page);
  1142. order = page_order(page);
  1143. /* Obey watermarks as if the page was being allocated */
  1144. watermark = low_wmark_pages(zone) + (1 << order);
  1145. if (!zone_watermark_ok(zone, 0, watermark, 0, 0))
  1146. return 0;
  1147. /* Remove page from free list */
  1148. list_del(&page->lru);
  1149. zone->free_area[order].nr_free--;
  1150. rmv_page_order(page);
  1151. __mod_zone_page_state(zone, NR_FREE_PAGES, -(1UL << order));
  1152. /* Split into individual pages */
  1153. set_page_refcounted(page);
  1154. split_page(page, order);
  1155. if (order >= pageblock_order - 1) {
  1156. struct page *endpage = page + (1 << order) - 1;
  1157. for (; page < endpage; page += pageblock_nr_pages)
  1158. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  1159. }
  1160. return 1 << order;
  1161. }
  1162. /*
  1163. * Really, prep_compound_page() should be called from __rmqueue_bulk(). But
  1164. * we cheat by calling it from here, in the order > 0 path. Saves a branch
  1165. * or two.
  1166. */
  1167. static inline
  1168. struct page *buffered_rmqueue(struct zone *preferred_zone,
  1169. struct zone *zone, int order, gfp_t gfp_flags,
  1170. int migratetype)
  1171. {
  1172. unsigned long flags;
  1173. struct page *page;
  1174. int cold = !!(gfp_flags & __GFP_COLD);
  1175. again:
  1176. if (likely(order == 0)) {
  1177. struct per_cpu_pages *pcp;
  1178. struct list_head *list;
  1179. local_irq_save(flags);
  1180. pcp = &this_cpu_ptr(zone->pageset)->pcp;
  1181. list = &pcp->lists[migratetype];
  1182. if (list_empty(list)) {
  1183. pcp->count += rmqueue_bulk(zone, 0,
  1184. pcp->batch, list,
  1185. migratetype, cold);
  1186. if (unlikely(list_empty(list)))
  1187. goto failed;
  1188. }
  1189. if (cold)
  1190. page = list_entry(list->prev, struct page, lru);
  1191. else
  1192. page = list_entry(list->next, struct page, lru);
  1193. list_del(&page->lru);
  1194. pcp->count--;
  1195. } else {
  1196. if (unlikely(gfp_flags & __GFP_NOFAIL)) {
  1197. /*
  1198. * __GFP_NOFAIL is not to be used in new code.
  1199. *
  1200. * All __GFP_NOFAIL callers should be fixed so that they
  1201. * properly detect and handle allocation failures.
  1202. *
  1203. * We most definitely don't want callers attempting to
  1204. * allocate greater than order-1 page units with
  1205. * __GFP_NOFAIL.
  1206. */
  1207. WARN_ON_ONCE(order > 1);
  1208. }
  1209. spin_lock_irqsave(&zone->lock, flags);
  1210. page = __rmqueue(zone, order, migratetype);
  1211. spin_unlock(&zone->lock);
  1212. if (!page)
  1213. goto failed;
  1214. __mod_zone_page_state(zone, NR_FREE_PAGES, -(1 << order));
  1215. }
  1216. __count_zone_vm_events(PGALLOC, zone, 1 << order);
  1217. zone_statistics(preferred_zone, zone, gfp_flags);
  1218. local_irq_restore(flags);
  1219. VM_BUG_ON(bad_range(zone, page));
  1220. if (prep_new_page(page, order, gfp_flags))
  1221. goto again;
  1222. return page;
  1223. failed:
  1224. local_irq_restore(flags);
  1225. return NULL;
  1226. }
  1227. /* The ALLOC_WMARK bits are used as an index to zone->watermark */
  1228. #define ALLOC_WMARK_MIN WMARK_MIN
  1229. #define ALLOC_WMARK_LOW WMARK_LOW
  1230. #define ALLOC_WMARK_HIGH WMARK_HIGH
  1231. #define ALLOC_NO_WATERMARKS 0x04 /* don't check watermarks at all */
  1232. /* Mask to get the watermark bits */
  1233. #define ALLOC_WMARK_MASK (ALLOC_NO_WATERMARKS-1)
  1234. #define ALLOC_HARDER 0x10 /* try to alloc harder */
  1235. #define ALLOC_HIGH 0x20 /* __GFP_HIGH set */
  1236. #define ALLOC_CPUSET 0x40 /* check for correct cpuset */
  1237. #ifdef CONFIG_FAIL_PAGE_ALLOC
  1238. static struct {
  1239. struct fault_attr attr;
  1240. u32 ignore_gfp_highmem;
  1241. u32 ignore_gfp_wait;
  1242. u32 min_order;
  1243. } fail_page_alloc = {
  1244. .attr = FAULT_ATTR_INITIALIZER,
  1245. .ignore_gfp_wait = 1,
  1246. .ignore_gfp_highmem = 1,
  1247. .min_order = 1,
  1248. };
  1249. static int __init setup_fail_page_alloc(char *str)
  1250. {
  1251. return setup_fault_attr(&fail_page_alloc.attr, str);
  1252. }
  1253. __setup("fail_page_alloc=", setup_fail_page_alloc);
  1254. static int should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
  1255. {
  1256. if (order < fail_page_alloc.min_order)
  1257. return 0;
  1258. if (gfp_mask & __GFP_NOFAIL)
  1259. return 0;
  1260. if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
  1261. return 0;
  1262. if (fail_page_alloc.ignore_gfp_wait && (gfp_mask & __GFP_WAIT))
  1263. return 0;
  1264. return should_fail(&fail_page_alloc.attr, 1 << order);
  1265. }
  1266. #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
  1267. static int __init fail_page_alloc_debugfs(void)
  1268. {
  1269. umode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
  1270. struct dentry *dir;
  1271. dir = fault_create_debugfs_attr("fail_page_alloc", NULL,
  1272. &fail_page_alloc.attr);
  1273. if (IS_ERR(dir))
  1274. return PTR_ERR(dir);
  1275. if (!debugfs_create_bool("ignore-gfp-wait", mode, dir,
  1276. &fail_page_alloc.ignore_gfp_wait))
  1277. goto fail;
  1278. if (!debugfs_create_bool("ignore-gfp-highmem", mode, dir,
  1279. &fail_page_alloc.ignore_gfp_highmem))
  1280. goto fail;
  1281. if (!debugfs_create_u32("min-order", mode, dir,
  1282. &fail_page_alloc.min_order))
  1283. goto fail;
  1284. return 0;
  1285. fail:
  1286. debugfs_remove_recursive(dir);
  1287. return -ENOMEM;
  1288. }
  1289. late_initcall(fail_page_alloc_debugfs);
  1290. #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
  1291. #else /* CONFIG_FAIL_PAGE_ALLOC */
  1292. static inline int should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
  1293. {
  1294. return 0;
  1295. }
  1296. #endif /* CONFIG_FAIL_PAGE_ALLOC */
  1297. /*
  1298. * Return true if free pages are above 'mark'. This takes into account the order
  1299. * of the allocation.
  1300. */
  1301. static bool __zone_watermark_ok(struct zone *z, int order, unsigned long mark,
  1302. int classzone_idx, int alloc_flags, long free_pages)
  1303. {
  1304. /* free_pages my go negative - that's OK */
  1305. long min = mark;
  1306. int o;
  1307. free_pages -= (1 << order) - 1;
  1308. if (alloc_flags & ALLOC_HIGH)
  1309. min -= min / 2;
  1310. if (alloc_flags & ALLOC_HARDER)
  1311. min -= min / 4;
  1312. if (free_pages <= min + z->lowmem_reserve[classzone_idx])
  1313. return false;
  1314. for (o = 0; o < order; o++) {
  1315. /* At the next order, this order's pages become unavailable */
  1316. free_pages -= z->free_area[o].nr_free << o;
  1317. /* Require fewer higher order pages to be free */
  1318. min >>= 1;
  1319. if (free_pages <= min)
  1320. return false;
  1321. }
  1322. return true;
  1323. }
  1324. bool zone_watermark_ok(struct zone *z, int order, unsigned long mark,
  1325. int classzone_idx, int alloc_flags)
  1326. {
  1327. return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
  1328. zone_page_state(z, NR_FREE_PAGES));
  1329. }
  1330. bool zone_watermark_ok_safe(struct zone *z, int order, unsigned long mark,
  1331. int classzone_idx, int alloc_flags)
  1332. {
  1333. long free_pages = zone_page_state(z, NR_FREE_PAGES);
  1334. if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
  1335. free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
  1336. return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
  1337. free_pages);
  1338. }
  1339. #ifdef CONFIG_NUMA
  1340. /*
  1341. * zlc_setup - Setup for "zonelist cache". Uses cached zone data to
  1342. * skip over zones that are not allowed by the cpuset, or that have
  1343. * been recently (in last second) found to be nearly full. See further
  1344. * comments in mmzone.h. Reduces cache footprint of zonelist scans
  1345. * that have to skip over a lot of full or unallowed zones.
  1346. *
  1347. * If the zonelist cache is present in the passed in zonelist, then
  1348. * returns a pointer to the allowed node mask (either the current
  1349. * tasks mems_allowed, or node_states[N_HIGH_MEMORY].)
  1350. *
  1351. * If the zonelist cache is not available for this zonelist, does
  1352. * nothing and returns NULL.
  1353. *
  1354. * If the fullzones BITMAP in the zonelist cache is stale (more than
  1355. * a second since last zap'd) then we zap it out (clear its bits.)
  1356. *
  1357. * We hold off even calling zlc_setup, until after we've checked the
  1358. * first zone in the zonelist, on the theory that most allocations will
  1359. * be satisfied from that first zone, so best to examine that zone as
  1360. * quickly as we can.
  1361. */
  1362. static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
  1363. {
  1364. struct zonelist_cache *zlc; /* cached zonelist speedup info */
  1365. nodemask_t *allowednodes; /* zonelist_cache approximation */
  1366. zlc = zonelist->zlcache_ptr;
  1367. if (!zlc)
  1368. return NULL;
  1369. if (time_after(jiffies, zlc->last_full_zap + HZ)) {
  1370. bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
  1371. zlc->last_full_zap = jiffies;
  1372. }
  1373. allowednodes = !in_interrupt() && (alloc_flags & ALLOC_CPUSET) ?
  1374. &cpuset_current_mems_allowed :
  1375. &node_states[N_HIGH_MEMORY];
  1376. return allowednodes;
  1377. }
  1378. /*
  1379. * Given 'z' scanning a zonelist, run a couple of quick checks to see
  1380. * if it is worth looking at further for free memory:
  1381. * 1) Check that the zone isn't thought to be full (doesn't have its
  1382. * bit set in the zonelist_cache fullzones BITMAP).
  1383. * 2) Check that the zones node (obtained from the zonelist_cache
  1384. * z_to_n[] mapping) is allowed in the passed in allowednodes mask.
  1385. * Return true (non-zero) if zone is worth looking at further, or
  1386. * else return false (zero) if it is not.
  1387. *
  1388. * This check -ignores- the distinction between various watermarks,
  1389. * such as GFP_HIGH, GFP_ATOMIC, PF_MEMALLOC, ... If a zone is
  1390. * found to be full for any variation of these watermarks, it will
  1391. * be considered full for up to one second by all requests, unless
  1392. * we are so low on memory on all allowed nodes that we are forced
  1393. * into the second scan of the zonelist.
  1394. *
  1395. * In the second scan we ignore this zonelist cache and exactly
  1396. * apply the watermarks to all zones, even it is slower to do so.
  1397. * We are low on memory in the second scan, and should leave no stone
  1398. * unturned looking for a free page.
  1399. */
  1400. static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z,
  1401. nodemask_t *allowednodes)
  1402. {
  1403. struct zonelist_cache *zlc; /* cached zonelist speedup info */
  1404. int i; /* index of *z in zonelist zones */
  1405. int n; /* node that zone *z is on */
  1406. zlc = zonelist->zlcache_ptr;
  1407. if (!zlc)
  1408. return 1;
  1409. i = z - zonelist->_zonerefs;
  1410. n = zlc->z_to_n[i];
  1411. /* This zone is worth trying if it is allowed but not full */
  1412. return node_isset(n, *allowednodes) && !test_bit(i, zlc->fullzones);
  1413. }
  1414. /*
  1415. * Given 'z' scanning a zonelist, set the corresponding bit in
  1416. * zlc->fullzones, so that subsequent attempts to allocate a page
  1417. * from that zone don't waste time re-examining it.
  1418. */
  1419. static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z)
  1420. {
  1421. struct zonelist_cache *zlc; /* cached zonelist speedup info */
  1422. int i; /* index of *z in zonelist zones */
  1423. zlc = zonelist->zlcache_ptr;
  1424. if (!zlc)
  1425. return;
  1426. i = z - zonelist->_zonerefs;
  1427. set_bit(i, zlc->fullzones);
  1428. }
  1429. /*
  1430. * clear all zones full, called after direct reclaim makes progress so that
  1431. * a zone that was recently full is not skipped over for up to a second
  1432. */
  1433. static void zlc_clear_zones_full(struct zonelist *zonelist)
  1434. {
  1435. struct zonelist_cache *zlc; /* cached zonelist speedup info */
  1436. zlc = zonelist->zlcache_ptr;
  1437. if (!zlc)
  1438. return;
  1439. bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
  1440. }
  1441. #else /* CONFIG_NUMA */
  1442. static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
  1443. {
  1444. return NULL;
  1445. }
  1446. static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z,
  1447. nodemask_t *allowednodes)
  1448. {
  1449. return 1;
  1450. }
  1451. static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z)
  1452. {
  1453. }
  1454. static void zlc_clear_zones_full(struct zonelist *zonelist)
  1455. {
  1456. }
  1457. #endif /* CONFIG_NUMA */
  1458. /*
  1459. * get_page_from_freelist goes through the zonelist trying to allocate
  1460. * a page.
  1461. */
  1462. static struct page *
  1463. get_page_from_freelist(gfp_t gfp_mask, nodemask_t *nodemask, unsigned int order,
  1464. struct zonelist *zonelist, int high_zoneidx, int alloc_flags,
  1465. struct zone *preferred_zone, int migratetype)
  1466. {
  1467. struct zoneref *z;
  1468. struct page *page = NULL;
  1469. int classzone_idx;
  1470. struct zone *zone;
  1471. nodemask_t *allowednodes = NULL;/* zonelist_cache approximation */
  1472. int zlc_active = 0; /* set if using zonelist_cache */
  1473. int did_zlc_setup = 0; /* just call zlc_setup() one time */
  1474. classzone_idx = zone_idx(preferred_zone);
  1475. zonelist_scan:
  1476. /*
  1477. * Scan zonelist, looking for a zone with enough free.
  1478. * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
  1479. */
  1480. for_each_zone_zonelist_nodemask(zone, z, zonelist,
  1481. high_zoneidx, nodemask) {
  1482. if (NUMA_BUILD && zlc_active &&
  1483. !zlc_zone_worth_trying(zonelist, z, allowednodes))
  1484. continue;
  1485. if ((alloc_flags & ALLOC_CPUSET) &&
  1486. !cpuset_zone_allowed_softwall(zone, gfp_mask))
  1487. continue;
  1488. /*
  1489. * When allocating a page cache page for writing, we
  1490. * want to get it from a zone that is within its dirty
  1491. * limit, such that no single zone holds more than its
  1492. * proportional share of globally allowed dirty pages.
  1493. * The dirty limits take into account the zone's
  1494. * lowmem reserves and high watermark so that kswapd
  1495. * should be able to balance it without having to
  1496. * write pages from its LRU list.
  1497. *
  1498. * This may look like it could increase pressure on
  1499. * lower zones by failing allocations in higher zones
  1500. * before they are full. But the pages that do spill
  1501. * over are limited as the lower zones are protected
  1502. * by this very same mechanism. It should not become
  1503. * a practical burden to them.
  1504. *
  1505. * XXX: For now, allow allocations to potentially
  1506. * exceed the per-zone dirty limit in the slowpath
  1507. * (ALLOC_WMARK_LOW unset) before going into reclaim,
  1508. * which is important when on a NUMA setup the allowed
  1509. * zones are together not big enough to reach the
  1510. * global limit. The proper fix for these situations
  1511. * will require awareness of zones in the
  1512. * dirty-throttling and the flusher threads.
  1513. */
  1514. if ((alloc_flags & ALLOC_WMARK_LOW) &&
  1515. (gfp_mask & __GFP_WRITE) && !zone_dirty_ok(zone))
  1516. goto this_zone_full;
  1517. BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
  1518. if (!(alloc_flags & ALLOC_NO_WATERMARKS)) {
  1519. unsigned long mark;
  1520. int ret;
  1521. mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK];
  1522. if (zone_watermark_ok(zone, order, mark,
  1523. classzone_idx, alloc_flags))
  1524. goto try_this_zone;
  1525. if (NUMA_BUILD && !did_zlc_setup && nr_online_nodes > 1) {
  1526. /*
  1527. * we do zlc_setup if there are multiple nodes
  1528. * and before considering the first zone allowed
  1529. * by the cpuset.
  1530. */
  1531. allowednodes = zlc_setup(zonelist, alloc_flags);
  1532. zlc_active = 1;
  1533. did_zlc_setup = 1;
  1534. }
  1535. if (zone_reclaim_mode == 0)
  1536. goto this_zone_full;
  1537. /*
  1538. * As we may have just activated ZLC, check if the first
  1539. * eligible zone has failed zone_reclaim recently.
  1540. */
  1541. if (NUMA_BUILD && zlc_active &&
  1542. !zlc_zone_worth_trying(zonelist, z, allowednodes))
  1543. continue;
  1544. ret = zone_reclaim(zone, gfp_mask, order);
  1545. switch (ret) {
  1546. case ZONE_RECLAIM_NOSCAN:
  1547. /* did not scan */
  1548. continue;
  1549. case ZONE_RECLAIM_FULL:
  1550. /* scanned but unreclaimable */
  1551. continue;
  1552. default:
  1553. /* did we reclaim enough */
  1554. if (!zone_watermark_ok(zone, order, mark,
  1555. classzone_idx, alloc_flags))
  1556. goto this_zone_full;
  1557. }
  1558. }
  1559. try_this_zone:
  1560. page = buffered_rmqueue(preferred_zone, zone, order,
  1561. gfp_mask, migratetype);
  1562. if (page)
  1563. break;
  1564. this_zone_full:
  1565. if (NUMA_BUILD)
  1566. zlc_mark_zone_full(zonelist, z);
  1567. }
  1568. if (unlikely(NUMA_BUILD && page == NULL && zlc_active)) {
  1569. /* Disable zlc cache for second zonelist scan */
  1570. zlc_active = 0;
  1571. goto zonelist_scan;
  1572. }
  1573. return page;
  1574. }
  1575. /*
  1576. * Large machines with many possible nodes should not always dump per-node
  1577. * meminfo in irq context.
  1578. */
  1579. static inline bool should_suppress_show_mem(void)
  1580. {
  1581. bool ret = false;
  1582. #if NODES_SHIFT > 8
  1583. ret = in_interrupt();
  1584. #endif
  1585. return ret;
  1586. }
  1587. static DEFINE_RATELIMIT_STATE(nopage_rs,
  1588. DEFAULT_RATELIMIT_INTERVAL,
  1589. DEFAULT_RATELIMIT_BURST);
  1590. void warn_alloc_failed(gfp_t gfp_mask, int order, const char *fmt, ...)
  1591. {
  1592. unsigned int filter = SHOW_MEM_FILTER_NODES;
  1593. if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) ||
  1594. debug_guardpage_minorder() > 0)
  1595. return;
  1596. /*
  1597. * This documents exceptions given to allocations in certain
  1598. * contexts that are allowed to allocate outside current's set
  1599. * of allowed nodes.
  1600. */
  1601. if (!(gfp_mask & __GFP_NOMEMALLOC))
  1602. if (test_thread_flag(TIF_MEMDIE) ||
  1603. (current->flags & (PF_MEMALLOC | PF_EXITING)))
  1604. filter &= ~SHOW_MEM_FILTER_NODES;
  1605. if (in_interrupt() || !(gfp_mask & __GFP_WAIT))
  1606. filter &= ~SHOW_MEM_FILTER_NODES;
  1607. if (fmt) {
  1608. struct va_format vaf;
  1609. va_list args;
  1610. va_start(args, fmt);
  1611. vaf.fmt = fmt;
  1612. vaf.va = &args;
  1613. pr_warn("%pV", &vaf);
  1614. va_end(args);
  1615. }
  1616. pr_warn("%s: page allocation failure: order:%d, mode:0x%x\n",
  1617. current->comm, order, gfp_mask);
  1618. dump_stack();
  1619. if (!should_suppress_show_mem())
  1620. show_mem(filter);
  1621. }
  1622. static inline int
  1623. should_alloc_retry(gfp_t gfp_mask, unsigned int order,
  1624. unsigned long did_some_progress,
  1625. unsigned long pages_reclaimed)
  1626. {
  1627. /* Do not loop if specifically requested */
  1628. if (gfp_mask & __GFP_NORETRY)
  1629. return 0;
  1630. /* Always retry if specifically requested */
  1631. if (gfp_mask & __GFP_NOFAIL)
  1632. return 1;
  1633. /*
  1634. * Suspend converts GFP_KERNEL to __GFP_WAIT which can prevent reclaim
  1635. * making forward progress without invoking OOM. Suspend also disables
  1636. * storage devices so kswapd will not help. Bail if we are suspending.
  1637. */
  1638. if (!did_some_progress && pm_suspended_storage())
  1639. return 0;
  1640. /*
  1641. * In this implementation, order <= PAGE_ALLOC_COSTLY_ORDER
  1642. * means __GFP_NOFAIL, but that may not be true in other
  1643. * implementations.
  1644. */
  1645. if (order <= PAGE_ALLOC_COSTLY_ORDER)
  1646. return 1;
  1647. /*
  1648. * For order > PAGE_ALLOC_COSTLY_ORDER, if __GFP_REPEAT is
  1649. * specified, then we retry until we no longer reclaim any pages
  1650. * (above), or we've reclaimed an order of pages at least as
  1651. * large as the allocation's order. In both cases, if the
  1652. * allocation still fails, we stop retrying.
  1653. */
  1654. if (gfp_mask & __GFP_REPEAT && pages_reclaimed < (1 << order))
  1655. return 1;
  1656. return 0;
  1657. }
  1658. static inline struct page *
  1659. __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
  1660. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1661. nodemask_t *nodemask, struct zone *preferred_zone,
  1662. int migratetype)
  1663. {
  1664. struct page *page;
  1665. /* Acquire the OOM killer lock for the zones in zonelist */
  1666. if (!try_set_zonelist_oom(zonelist, gfp_mask)) {
  1667. schedule_timeout_uninterruptible(1);
  1668. return NULL;
  1669. }
  1670. /*
  1671. * Go through the zonelist yet one more time, keep very high watermark
  1672. * here, this is only to catch a parallel oom killing, we must fail if
  1673. * we're still under heavy pressure.
  1674. */
  1675. page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask,
  1676. order, zonelist, high_zoneidx,
  1677. ALLOC_WMARK_HIGH|ALLOC_CPUSET,
  1678. preferred_zone, migratetype);
  1679. if (page)
  1680. goto out;
  1681. if (!(gfp_mask & __GFP_NOFAIL)) {
  1682. /* The OOM killer will not help higher order allocs */
  1683. if (order > PAGE_ALLOC_COSTLY_ORDER)
  1684. goto out;
  1685. /* The OOM killer does not needlessly kill tasks for lowmem */
  1686. if (high_zoneidx < ZONE_NORMAL)
  1687. goto out;
  1688. /*
  1689. * GFP_THISNODE contains __GFP_NORETRY and we never hit this.
  1690. * Sanity check for bare calls of __GFP_THISNODE, not real OOM.
  1691. * The caller should handle page allocation failure by itself if
  1692. * it specifies __GFP_THISNODE.
  1693. * Note: Hugepage uses it but will hit PAGE_ALLOC_COSTLY_ORDER.
  1694. */
  1695. if (gfp_mask & __GFP_THISNODE)
  1696. goto out;
  1697. }
  1698. /* Exhausted what can be done so it's blamo time */
  1699. out_of_memory(zonelist, gfp_mask, order, nodemask, false);
  1700. out:
  1701. clear_zonelist_oom(zonelist, gfp_mask);
  1702. return page;
  1703. }
  1704. #ifdef CONFIG_COMPACTION
  1705. /* Try memory compaction for high-order allocations before reclaim */
  1706. static struct page *
  1707. __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
  1708. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1709. nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
  1710. int migratetype, bool sync_migration,
  1711. bool *deferred_compaction,
  1712. unsigned long *did_some_progress)
  1713. {
  1714. struct page *page;
  1715. if (!order)
  1716. return NULL;
  1717. if (compaction_deferred(preferred_zone, order)) {
  1718. *deferred_compaction = true;
  1719. return NULL;
  1720. }
  1721. current->flags |= PF_MEMALLOC;
  1722. *did_some_progress = try_to_compact_pages(zonelist, order, gfp_mask,
  1723. nodemask, sync_migration);
  1724. current->flags &= ~PF_MEMALLOC;
  1725. if (*did_some_progress != COMPACT_SKIPPED) {
  1726. /* Page migration frees to the PCP lists but we want merging */
  1727. drain_pages(get_cpu());
  1728. put_cpu();
  1729. page = get_page_from_freelist(gfp_mask, nodemask,
  1730. order, zonelist, high_zoneidx,
  1731. alloc_flags, preferred_zone,
  1732. migratetype);
  1733. if (page) {
  1734. preferred_zone->compact_considered = 0;
  1735. preferred_zone->compact_defer_shift = 0;
  1736. if (order >= preferred_zone->compact_order_failed)
  1737. preferred_zone->compact_order_failed = order + 1;
  1738. count_vm_event(COMPACTSUCCESS);
  1739. return page;
  1740. }
  1741. /*
  1742. * It's bad if compaction run occurs and fails.
  1743. * The most likely reason is that pages exist,
  1744. * but not enough to satisfy watermarks.
  1745. */
  1746. count_vm_event(COMPACTFAIL);
  1747. /*
  1748. * As async compaction considers a subset of pageblocks, only
  1749. * defer if the failure was a sync compaction failure.
  1750. */
  1751. if (sync_migration)
  1752. defer_compaction(preferred_zone, order);
  1753. cond_resched();
  1754. }
  1755. return NULL;
  1756. }
  1757. #else
  1758. static inline struct page *
  1759. __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
  1760. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1761. nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
  1762. int migratetype, bool sync_migration,
  1763. bool *deferred_compaction,
  1764. unsigned long *did_some_progress)
  1765. {
  1766. return NULL;
  1767. }
  1768. #endif /* CONFIG_COMPACTION */
  1769. /* The really slow allocator path where we enter direct reclaim */
  1770. static inline struct page *
  1771. __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
  1772. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1773. nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
  1774. int migratetype, unsigned long *did_some_progress)
  1775. {
  1776. struct page *page = NULL;
  1777. struct reclaim_state reclaim_state;
  1778. bool drained = false;
  1779. cond_resched();
  1780. /* We now go into synchronous reclaim */
  1781. cpuset_memory_pressure_bump();
  1782. current->flags |= PF_MEMALLOC;
  1783. lockdep_set_current_reclaim_state(gfp_mask);
  1784. reclaim_state.reclaimed_slab = 0;
  1785. current->reclaim_state = &reclaim_state;
  1786. *did_some_progress = try_to_free_pages(zonelist, order, gfp_mask, nodemask);
  1787. current->reclaim_state = NULL;
  1788. lockdep_clear_current_reclaim_state();
  1789. current->flags &= ~PF_MEMALLOC;
  1790. cond_resched();
  1791. if (unlikely(!(*did_some_progress)))
  1792. return NULL;
  1793. /* After successful reclaim, reconsider all zones for allocation */
  1794. if (NUMA_BUILD)
  1795. zlc_clear_zones_full(zonelist);
  1796. retry:
  1797. page = get_page_from_freelist(gfp_mask, nodemask, order,
  1798. zonelist, high_zoneidx,
  1799. alloc_flags, preferred_zone,
  1800. migratetype);
  1801. /*
  1802. * If an allocation failed after direct reclaim, it could be because
  1803. * pages are pinned on the per-cpu lists. Drain them and try again
  1804. */
  1805. if (!page && !drained) {
  1806. drain_all_pages();
  1807. drained = true;
  1808. goto retry;
  1809. }
  1810. return page;
  1811. }
  1812. /*
  1813. * This is called in the allocator slow-path if the allocation request is of
  1814. * sufficient urgency to ignore watermarks and take other desperate measures
  1815. */
  1816. static inline struct page *
  1817. __alloc_pages_high_priority(gfp_t gfp_mask, unsigned int order,
  1818. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1819. nodemask_t *nodemask, struct zone *preferred_zone,
  1820. int migratetype)
  1821. {
  1822. struct page *page;
  1823. do {
  1824. page = get_page_from_freelist(gfp_mask, nodemask, order,
  1825. zonelist, high_zoneidx, ALLOC_NO_WATERMARKS,
  1826. preferred_zone, migratetype);
  1827. if (!page && gfp_mask & __GFP_NOFAIL)
  1828. wait_iff_congested(preferred_zone, BLK_RW_ASYNC, HZ/50);
  1829. } while (!page && (gfp_mask & __GFP_NOFAIL));
  1830. return page;
  1831. }
  1832. static inline
  1833. void wake_all_kswapd(unsigned int order, struct zonelist *zonelist,
  1834. enum zone_type high_zoneidx,
  1835. enum zone_type classzone_idx)
  1836. {
  1837. struct zoneref *z;
  1838. struct zone *zone;
  1839. for_each_zone_zonelist(zone, z, zonelist, high_zoneidx)
  1840. wakeup_kswapd(zone, order, classzone_idx);
  1841. }
  1842. static inline int
  1843. gfp_to_alloc_flags(gfp_t gfp_mask)
  1844. {
  1845. int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
  1846. const gfp_t wait = gfp_mask & __GFP_WAIT;
  1847. /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */
  1848. BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);
  1849. /*
  1850. * The caller may dip into page reserves a bit more if the caller
  1851. * cannot run direct reclaim, or if the caller has realtime scheduling
  1852. * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will
  1853. * set both ALLOC_HARDER (!wait) and ALLOC_HIGH (__GFP_HIGH).
  1854. */
  1855. alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH);
  1856. if (!wait) {
  1857. /*
  1858. * Not worth trying to allocate harder for
  1859. * __GFP_NOMEMALLOC even if it can't schedule.
  1860. */
  1861. if (!(gfp_mask & __GFP_NOMEMALLOC))
  1862. alloc_flags |= ALLOC_HARDER;
  1863. /*
  1864. * Ignore cpuset if GFP_ATOMIC (!wait) rather than fail alloc.
  1865. * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
  1866. */
  1867. alloc_flags &= ~ALLOC_CPUSET;
  1868. } else if (unlikely(rt_task(current)) && !in_interrupt())
  1869. alloc_flags |= ALLOC_HARDER;
  1870. if (likely(!(gfp_mask & __GFP_NOMEMALLOC))) {
  1871. if (!in_interrupt() &&
  1872. ((current->flags & PF_MEMALLOC) ||
  1873. unlikely(test_thread_flag(TIF_MEMDIE))))
  1874. alloc_flags |= ALLOC_NO_WATERMARKS;
  1875. }
  1876. return alloc_flags;
  1877. }
  1878. static inline struct page *
  1879. __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
  1880. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1881. nodemask_t *nodemask, struct zone *preferred_zone,
  1882. int migratetype)
  1883. {
  1884. const gfp_t wait = gfp_mask & __GFP_WAIT;
  1885. struct page *page = NULL;
  1886. int alloc_flags;
  1887. unsigned long pages_reclaimed = 0;
  1888. unsigned long did_some_progress;
  1889. bool sync_migration = false;
  1890. bool deferred_compaction = false;
  1891. /*
  1892. * In the slowpath, we sanity check order to avoid ever trying to
  1893. * reclaim >= MAX_ORDER areas which will never succeed. Callers may
  1894. * be using allocators in order of preference for an area that is
  1895. * too large.
  1896. */
  1897. if (order >= MAX_ORDER) {
  1898. WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));
  1899. return NULL;
  1900. }
  1901. /*
  1902. * GFP_THISNODE (meaning __GFP_THISNODE, __GFP_NORETRY and
  1903. * __GFP_NOWARN set) should not cause reclaim since the subsystem
  1904. * (f.e. slab) using GFP_THISNODE may choose to trigger reclaim
  1905. * using a larger set of nodes after it has established that the
  1906. * allowed per node queues are empty and that nodes are
  1907. * over allocated.
  1908. */
  1909. if (NUMA_BUILD && (gfp_mask & GFP_THISNODE) == GFP_THISNODE)
  1910. goto nopage;
  1911. restart:
  1912. if (!(gfp_mask & __GFP_NO_KSWAPD))
  1913. wake_all_kswapd(order, zonelist, high_zoneidx,
  1914. zone_idx(preferred_zone));
  1915. /*
  1916. * OK, we're below the kswapd watermark and have kicked background
  1917. * reclaim. Now things get more complex, so set up alloc_flags according
  1918. * to how we want to proceed.
  1919. */
  1920. alloc_flags = gfp_to_alloc_flags(gfp_mask);
  1921. /*
  1922. * Find the true preferred zone if the allocation is unconstrained by
  1923. * cpusets.
  1924. */
  1925. if (!(alloc_flags & ALLOC_CPUSET) && !nodemask)
  1926. first_zones_zonelist(zonelist, high_zoneidx, NULL,
  1927. &preferred_zone);
  1928. rebalance:
  1929. /* This is the last chance, in general, before the goto nopage. */
  1930. page = get_page_from_freelist(gfp_mask, nodemask, order, zonelist,
  1931. high_zoneidx, alloc_flags & ~ALLOC_NO_WATERMARKS,
  1932. preferred_zone, migratetype);
  1933. if (page)
  1934. goto got_pg;
  1935. /* Allocate without watermarks if the context allows */
  1936. if (alloc_flags & ALLOC_NO_WATERMARKS) {
  1937. page = __alloc_pages_high_priority(gfp_mask, order,
  1938. zonelist, high_zoneidx, nodemask,
  1939. preferred_zone, migratetype);
  1940. if (page)
  1941. goto got_pg;
  1942. }
  1943. /* Atomic allocations - we can't balance anything */
  1944. if (!wait)
  1945. goto nopage;
  1946. /* Avoid recursion of direct reclaim */
  1947. if (current->flags & PF_MEMALLOC)
  1948. goto nopage;
  1949. /* Avoid allocations with no watermarks from looping endlessly */
  1950. if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL))
  1951. goto nopage;
  1952. /*
  1953. * Try direct compaction. The first pass is asynchronous. Subsequent
  1954. * attempts after direct reclaim are synchronous
  1955. */
  1956. page = __alloc_pages_direct_compact(gfp_mask, order,
  1957. zonelist, high_zoneidx,
  1958. nodemask,
  1959. alloc_flags, preferred_zone,
  1960. migratetype, sync_migration,
  1961. &deferred_compaction,
  1962. &did_some_progress);
  1963. if (page)
  1964. goto got_pg;
  1965. sync_migration = true;
  1966. /*
  1967. * If compaction is deferred for high-order allocations, it is because
  1968. * sync compaction recently failed. In this is the case and the caller
  1969. * has requested the system not be heavily disrupted, fail the
  1970. * allocation now instead of entering direct reclaim
  1971. */
  1972. if (deferred_compaction && (gfp_mask & __GFP_NO_KSWAPD))
  1973. goto nopage;
  1974. /* Try direct reclaim and then allocating */
  1975. page = __alloc_pages_direct_reclaim(gfp_mask, order,
  1976. zonelist, high_zoneidx,
  1977. nodemask,
  1978. alloc_flags, preferred_zone,
  1979. migratetype, &did_some_progress);
  1980. if (page)
  1981. goto got_pg;
  1982. /*
  1983. * If we failed to make any progress reclaiming, then we are
  1984. * running out of options and have to consider going OOM
  1985. */
  1986. if (!did_some_progress) {
  1987. if ((gfp_mask & __GFP_FS) && !(gfp_mask & __GFP_NORETRY)) {
  1988. if (oom_killer_disabled)
  1989. goto nopage;
  1990. page = __alloc_pages_may_oom(gfp_mask, order,
  1991. zonelist, high_zoneidx,
  1992. nodemask, preferred_zone,
  1993. migratetype);
  1994. if (page)
  1995. goto got_pg;
  1996. if (!(gfp_mask & __GFP_NOFAIL)) {
  1997. /*
  1998. * The oom killer is not called for high-order
  1999. * allocations that may fail, so if no progress
  2000. * is being made, there are no other options and
  2001. * retrying is unlikely to help.
  2002. */
  2003. if (order > PAGE_ALLOC_COSTLY_ORDER)
  2004. goto nopage;
  2005. /*
  2006. * The oom killer is not called for lowmem
  2007. * allocations to prevent needlessly killing
  2008. * innocent tasks.
  2009. */
  2010. if (high_zoneidx < ZONE_NORMAL)
  2011. goto nopage;
  2012. }
  2013. goto restart;
  2014. }
  2015. }
  2016. /* Check if we should retry the allocation */
  2017. pages_reclaimed += did_some_progress;
  2018. if (should_alloc_retry(gfp_mask, order, did_some_progress,
  2019. pages_reclaimed)) {
  2020. /* Wait for some write requests to complete then retry */
  2021. wait_iff_congested(preferred_zone, BLK_RW_ASYNC, HZ/50);
  2022. goto rebalance;
  2023. } else {
  2024. /*
  2025. * High-order allocations do not necessarily loop after
  2026. * direct reclaim and reclaim/compaction depends on compaction
  2027. * being called after reclaim so call directly if necessary
  2028. */
  2029. page = __alloc_pages_direct_compact(gfp_mask, order,
  2030. zonelist, high_zoneidx,
  2031. nodemask,
  2032. alloc_flags, preferred_zone,
  2033. migratetype, sync_migration,
  2034. &deferred_compaction,
  2035. &did_some_progress);
  2036. if (page)
  2037. goto got_pg;
  2038. }
  2039. nopage:
  2040. warn_alloc_failed(gfp_mask, order, NULL);
  2041. return page;
  2042. got_pg:
  2043. if (kmemcheck_enabled)
  2044. kmemcheck_pagealloc_alloc(page, order, gfp_mask);
  2045. return page;
  2046. }
  2047. /*
  2048. * This is the 'heart' of the zoned buddy allocator.
  2049. */
  2050. struct page *
  2051. __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
  2052. struct zonelist *zonelist, nodemask_t *nodemask)
  2053. {
  2054. enum zone_type high_zoneidx = gfp_zone(gfp_mask);
  2055. struct zone *preferred_zone;
  2056. struct page *page;
  2057. int migratetype = allocflags_to_migratetype(gfp_mask);
  2058. gfp_mask &= gfp_allowed_mask;
  2059. lockdep_trace_alloc(gfp_mask);
  2060. might_sleep_if(gfp_mask & __GFP_WAIT);
  2061. if (should_fail_alloc_page(gfp_mask, order))
  2062. return NULL;
  2063. /*
  2064. * Check the zones suitable for the gfp_mask contain at least one
  2065. * valid zone. It's possible to have an empty zonelist as a result
  2066. * of GFP_THISNODE and a memoryless node
  2067. */
  2068. if (unlikely(!zonelist->_zonerefs->zone))
  2069. return NULL;
  2070. get_mems_allowed();
  2071. /* The preferred zone is used for statistics later */
  2072. first_zones_zonelist(zonelist, high_zoneidx,
  2073. nodemask ? : &cpuset_current_mems_allowed,
  2074. &preferred_zone);
  2075. if (!preferred_zone) {
  2076. put_mems_allowed();
  2077. return NULL;
  2078. }
  2079. /* First allocation attempt */
  2080. page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask, order,
  2081. zonelist, high_zoneidx, ALLOC_WMARK_LOW|ALLOC_CPUSET,
  2082. preferred_zone, migratetype);
  2083. if (unlikely(!page))
  2084. page = __alloc_pages_slowpath(gfp_mask, order,
  2085. zonelist, high_zoneidx, nodemask,
  2086. preferred_zone, migratetype);
  2087. put_mems_allowed();
  2088. trace_mm_page_alloc(page, order, gfp_mask, migratetype);
  2089. return page;
  2090. }
  2091. EXPORT_SYMBOL(__alloc_pages_nodemask);
  2092. /*
  2093. * Common helper functions.
  2094. */
  2095. unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
  2096. {
  2097. struct page *page;
  2098. /*
  2099. * __get_free_pages() returns a 32-bit address, which cannot represent
  2100. * a highmem page
  2101. */
  2102. VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0);
  2103. page = alloc_pages(gfp_mask, order);
  2104. if (!page)
  2105. return 0;
  2106. return (unsigned long) page_address(page);
  2107. }
  2108. EXPORT_SYMBOL(__get_free_pages);
  2109. unsigned long get_zeroed_page(gfp_t gfp_mask)
  2110. {
  2111. return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
  2112. }
  2113. EXPORT_SYMBOL(get_zeroed_page);
  2114. void __free_pages(struct page *page, unsigned int order)
  2115. {
  2116. if (put_page_testzero(page)) {
  2117. if (order == 0)
  2118. free_hot_cold_page(page, 0);
  2119. else
  2120. __free_pages_ok(page, order);
  2121. }
  2122. }
  2123. EXPORT_SYMBOL(__free_pages);
  2124. void free_pages(unsigned long addr, unsigned int order)
  2125. {
  2126. if (addr != 0) {
  2127. VM_BUG_ON(!virt_addr_valid((void *)addr));
  2128. __free_pages(virt_to_page((void *)addr), order);
  2129. }
  2130. }
  2131. EXPORT_SYMBOL(free_pages);
  2132. static void *make_alloc_exact(unsigned long addr, unsigned order, size_t size)
  2133. {
  2134. if (addr) {
  2135. unsigned long alloc_end = addr + (PAGE_SIZE << order);
  2136. unsigned long used = addr + PAGE_ALIGN(size);
  2137. split_page(virt_to_page((void *)addr), order);
  2138. while (used < alloc_end) {
  2139. free_page(used);
  2140. used += PAGE_SIZE;
  2141. }
  2142. }
  2143. return (void *)addr;
  2144. }
  2145. /**
  2146. * alloc_pages_exact - allocate an exact number physically-contiguous pages.
  2147. * @size: the number of bytes to allocate
  2148. * @gfp_mask: GFP flags for the allocation
  2149. *
  2150. * This function is similar to alloc_pages(), except that it allocates the
  2151. * minimum number of pages to satisfy the request. alloc_pages() can only
  2152. * allocate memory in power-of-two pages.
  2153. *
  2154. * This function is also limited by MAX_ORDER.
  2155. *
  2156. * Memory allocated by this function must be released by free_pages_exact().
  2157. */
  2158. void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
  2159. {
  2160. unsigned int order = get_order(size);
  2161. unsigned long addr;
  2162. addr = __get_free_pages(gfp_mask, order);
  2163. return make_alloc_exact(addr, order, size);
  2164. }
  2165. EXPORT_SYMBOL(alloc_pages_exact);
  2166. /**
  2167. * alloc_pages_exact_nid - allocate an exact number of physically-contiguous
  2168. * pages on a node.
  2169. * @nid: the preferred node ID where memory should be allocated
  2170. * @size: the number of bytes to allocate
  2171. * @gfp_mask: GFP flags for the allocation
  2172. *
  2173. * Like alloc_pages_exact(), but try to allocate on node nid first before falling
  2174. * back.
  2175. * Note this is not alloc_pages_exact_node() which allocates on a specific node,
  2176. * but is not exact.
  2177. */
  2178. void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask)
  2179. {
  2180. unsigned order = get_order(size);
  2181. struct page *p = alloc_pages_node(nid, gfp_mask, order);
  2182. if (!p)
  2183. return NULL;
  2184. return make_alloc_exact((unsigned long)page_address(p), order, size);
  2185. }
  2186. EXPORT_SYMBOL(alloc_pages_exact_nid);
  2187. /**
  2188. * free_pages_exact - release memory allocated via alloc_pages_exact()
  2189. * @virt: the value returned by alloc_pages_exact.
  2190. * @size: size of allocation, same value as passed to alloc_pages_exact().
  2191. *
  2192. * Release the memory allocated by a previous call to alloc_pages_exact.
  2193. */
  2194. void free_pages_exact(void *virt, size_t size)
  2195. {
  2196. unsigned long addr = (unsigned long)virt;
  2197. unsigned long end = addr + PAGE_ALIGN(size);
  2198. while (addr < end) {
  2199. free_page(addr);
  2200. addr += PAGE_SIZE;
  2201. }
  2202. }
  2203. EXPORT_SYMBOL(free_pages_exact);
  2204. static unsigned int nr_free_zone_pages(int offset)
  2205. {
  2206. struct zoneref *z;
  2207. struct zone *zone;
  2208. /* Just pick one node, since fallback list is circular */
  2209. unsigned int sum = 0;
  2210. struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
  2211. for_each_zone_zonelist(zone, z, zonelist, offset) {
  2212. unsigned long size = zone->present_pages;
  2213. unsigned long high = high_wmark_pages(zone);
  2214. if (size > high)
  2215. sum += size - high;
  2216. }
  2217. return sum;
  2218. }
  2219. /*
  2220. * Amount of free RAM allocatable within ZONE_DMA and ZONE_NORMAL
  2221. */
  2222. unsigned int nr_free_buffer_pages(void)
  2223. {
  2224. return nr_free_zone_pages(gfp_zone(GFP_USER));
  2225. }
  2226. EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
  2227. /*
  2228. * Amount of free RAM allocatable within all zones
  2229. */
  2230. unsigned int nr_free_pagecache_pages(void)
  2231. {
  2232. return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
  2233. }
  2234. static inline void show_node(struct zone *zone)
  2235. {
  2236. if (NUMA_BUILD)
  2237. printk("Node %d ", zone_to_nid(zone));
  2238. }
  2239. void si_meminfo(struct sysinfo *val)
  2240. {
  2241. val->totalram = totalram_pages;
  2242. val->sharedram = 0;
  2243. val->freeram = global_page_state(NR_FREE_PAGES);
  2244. val->bufferram = nr_blockdev_pages();
  2245. val->totalhigh = totalhigh_pages;
  2246. val->freehigh = nr_free_highpages();
  2247. val->mem_unit = PAGE_SIZE;
  2248. }
  2249. EXPORT_SYMBOL(si_meminfo);
  2250. #ifdef CONFIG_NUMA
  2251. void si_meminfo_node(struct sysinfo *val, int nid)
  2252. {
  2253. pg_data_t *pgdat = NODE_DATA(nid);
  2254. val->totalram = pgdat->node_present_pages;
  2255. val->freeram = node_page_state(nid, NR_FREE_PAGES);
  2256. #ifdef CONFIG_HIGHMEM
  2257. val->totalhigh = pgdat->node_zones[ZONE_HIGHMEM].present_pages;
  2258. val->freehigh = zone_page_state(&pgdat->node_zones[ZONE_HIGHMEM],
  2259. NR_FREE_PAGES);
  2260. #else
  2261. val->totalhigh = 0;
  2262. val->freehigh = 0;
  2263. #endif
  2264. val->mem_unit = PAGE_SIZE;
  2265. }
  2266. #endif
  2267. /*
  2268. * Determine whether the node should be displayed or not, depending on whether
  2269. * SHOW_MEM_FILTER_NODES was passed to show_free_areas().
  2270. */
  2271. bool skip_free_areas_node(unsigned int flags, int nid)
  2272. {
  2273. bool ret = false;
  2274. if (!(flags & SHOW_MEM_FILTER_NODES))
  2275. goto out;
  2276. get_mems_allowed();
  2277. ret = !node_isset(nid, cpuset_current_mems_allowed);
  2278. put_mems_allowed();
  2279. out:
  2280. return ret;
  2281. }
  2282. #define K(x) ((x) << (PAGE_SHIFT-10))
  2283. /*
  2284. * Show free area list (used inside shift_scroll-lock stuff)
  2285. * We also calculate the percentage fragmentation. We do this by counting the
  2286. * memory on each free list with the exception of the first item on the list.
  2287. * Suppresses nodes that are not allowed by current's cpuset if
  2288. * SHOW_MEM_FILTER_NODES is passed.
  2289. */
  2290. void show_free_areas(unsigned int filter)
  2291. {
  2292. int cpu;
  2293. struct zone *zone;
  2294. for_each_populated_zone(zone) {
  2295. if (skip_free_areas_node(filter, zone_to_nid(zone)))
  2296. continue;
  2297. show_node(zone);
  2298. printk("%s per-cpu:\n", zone->name);
  2299. for_each_online_cpu(cpu) {
  2300. struct per_cpu_pageset *pageset;
  2301. pageset = per_cpu_ptr(zone->pageset, cpu);
  2302. printk("CPU %4d: hi:%5d, btch:%4d usd:%4d\n",
  2303. cpu, pageset->pcp.high,
  2304. pageset->pcp.batch, pageset->pcp.count);
  2305. }
  2306. }
  2307. printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n"
  2308. " active_file:%lu inactive_file:%lu isolated_file:%lu\n"
  2309. " unevictable:%lu"
  2310. " dirty:%lu writeback:%lu unstable:%lu\n"
  2311. " free:%lu slab_reclaimable:%lu slab_unreclaimable:%lu\n"
  2312. " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n",
  2313. global_page_state(NR_ACTIVE_ANON),
  2314. global_page_state(NR_INACTIVE_ANON),
  2315. global_page_state(NR_ISOLATED_ANON),
  2316. global_page_state(NR_ACTIVE_FILE),
  2317. global_page_state(NR_INACTIVE_FILE),
  2318. global_page_state(NR_ISOLATED_FILE),
  2319. global_page_state(NR_UNEVICTABLE),
  2320. global_page_state(NR_FILE_DIRTY),
  2321. global_page_state(NR_WRITEBACK),
  2322. global_page_state(NR_UNSTABLE_NFS),
  2323. global_page_state(NR_FREE_PAGES),
  2324. global_page_state(NR_SLAB_RECLAIMABLE),
  2325. global_page_state(NR_SLAB_UNRECLAIMABLE),
  2326. global_page_state(NR_FILE_MAPPED),
  2327. global_page_state(NR_SHMEM),
  2328. global_page_state(NR_PAGETABLE),
  2329. global_page_state(NR_BOUNCE));
  2330. for_each_populated_zone(zone) {
  2331. int i;
  2332. if (skip_free_areas_node(filter, zone_to_nid(zone)))
  2333. continue;
  2334. show_node(zone);
  2335. printk("%s"
  2336. " free:%lukB"
  2337. " min:%lukB"
  2338. " low:%lukB"
  2339. " high:%lukB"
  2340. " active_anon:%lukB"
  2341. " inactive_anon:%lukB"
  2342. " active_file:%lukB"
  2343. " inactive_file:%lukB"
  2344. " unevictable:%lukB"
  2345. " isolated(anon):%lukB"
  2346. " isolated(file):%lukB"
  2347. " present:%lukB"
  2348. " mlocked:%lukB"
  2349. " dirty:%lukB"
  2350. " writeback:%lukB"
  2351. " mapped:%lukB"
  2352. " shmem:%lukB"
  2353. " slab_reclaimable:%lukB"
  2354. " slab_unreclaimable:%lukB"
  2355. " kernel_stack:%lukB"
  2356. " pagetables:%lukB"
  2357. " unstable:%lukB"
  2358. " bounce:%lukB"
  2359. " writeback_tmp:%lukB"
  2360. " pages_scanned:%lu"
  2361. " all_unreclaimable? %s"
  2362. "\n",
  2363. zone->name,
  2364. K(zone_page_state(zone, NR_FREE_PAGES)),
  2365. K(min_wmark_pages(zone)),
  2366. K(low_wmark_pages(zone)),
  2367. K(high_wmark_pages(zone)),
  2368. K(zone_page_state(zone, NR_ACTIVE_ANON)),
  2369. K(zone_page_state(zone, NR_INACTIVE_ANON)),
  2370. K(zone_page_state(zone, NR_ACTIVE_FILE)),
  2371. K(zone_page_state(zone, NR_INACTIVE_FILE)),
  2372. K(zone_page_state(zone, NR_UNEVICTABLE)),
  2373. K(zone_page_state(zone, NR_ISOLATED_ANON)),
  2374. K(zone_page_state(zone, NR_ISOLATED_FILE)),
  2375. K(zone->present_pages),
  2376. K(zone_page_state(zone, NR_MLOCK)),
  2377. K(zone_page_state(zone, NR_FILE_DIRTY)),
  2378. K(zone_page_state(zone, NR_WRITEBACK)),
  2379. K(zone_page_state(zone, NR_FILE_MAPPED)),
  2380. K(zone_page_state(zone, NR_SHMEM)),
  2381. K(zone_page_state(zone, NR_SLAB_RECLAIMABLE)),
  2382. K(zone_page_state(zone, NR_SLAB_UNRECLAIMABLE)),
  2383. zone_page_state(zone, NR_KERNEL_STACK) *
  2384. THREAD_SIZE / 1024,
  2385. K(zone_page_state(zone, NR_PAGETABLE)),
  2386. K(zone_page_state(zone, NR_UNSTABLE_NFS)),
  2387. K(zone_page_state(zone, NR_BOUNCE)),
  2388. K(zone_page_state(zone, NR_WRITEBACK_TEMP)),
  2389. zone->pages_scanned,
  2390. (zone->all_unreclaimable ? "yes" : "no")
  2391. );
  2392. printk("lowmem_reserve[]:");
  2393. for (i = 0; i < MAX_NR_ZONES; i++)
  2394. printk(" %lu", zone->lowmem_reserve[i]);
  2395. printk("\n");
  2396. }
  2397. for_each_populated_zone(zone) {
  2398. unsigned long nr[MAX_ORDER], flags, order, total = 0;
  2399. if (skip_free_areas_node(filter, zone_to_nid(zone)))
  2400. continue;
  2401. show_node(zone);
  2402. printk("%s: ", zone->name);
  2403. spin_lock_irqsave(&zone->lock, flags);
  2404. for (order = 0; order < MAX_ORDER; order++) {
  2405. nr[order] = zone->free_area[order].nr_free;
  2406. total += nr[order] << order;
  2407. }
  2408. spin_unlock_irqrestore(&zone->lock, flags);
  2409. for (order = 0; order < MAX_ORDER; order++)
  2410. printk("%lu*%lukB ", nr[order], K(1UL) << order);
  2411. printk("= %lukB\n", K(total));
  2412. }
  2413. printk("%ld total pagecache pages\n", global_page_state(NR_FILE_PAGES));
  2414. show_swap_cache_info();
  2415. }
  2416. static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
  2417. {
  2418. zoneref->zone = zone;
  2419. zoneref->zone_idx = zone_idx(zone);
  2420. }
  2421. /*
  2422. * Builds allocation fallback zone lists.
  2423. *
  2424. * Add all populated zones of a node to the zonelist.
  2425. */
  2426. static int build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist,
  2427. int nr_zones, enum zone_type zone_type)
  2428. {
  2429. struct zone *zone;
  2430. BUG_ON(zone_type >= MAX_NR_ZONES);
  2431. zone_type++;
  2432. do {
  2433. zone_type--;
  2434. zone = pgdat->node_zones + zone_type;
  2435. if (populated_zone(zone)) {
  2436. zoneref_set_zone(zone,
  2437. &zonelist->_zonerefs[nr_zones++]);
  2438. check_highest_zone(zone_type);
  2439. }
  2440. } while (zone_type);
  2441. return nr_zones;
  2442. }
  2443. /*
  2444. * zonelist_order:
  2445. * 0 = automatic detection of better ordering.
  2446. * 1 = order by ([node] distance, -zonetype)
  2447. * 2 = order by (-zonetype, [node] distance)
  2448. *
  2449. * If not NUMA, ZONELIST_ORDER_ZONE and ZONELIST_ORDER_NODE will create
  2450. * the same zonelist. So only NUMA can configure this param.
  2451. */
  2452. #define ZONELIST_ORDER_DEFAULT 0
  2453. #define ZONELIST_ORDER_NODE 1
  2454. #define ZONELIST_ORDER_ZONE 2
  2455. /* zonelist order in the kernel.
  2456. * set_zonelist_order() will set this to NODE or ZONE.
  2457. */
  2458. static int current_zonelist_order = ZONELIST_ORDER_DEFAULT;
  2459. static char zonelist_order_name[3][8] = {"Default", "Node", "Zone"};
  2460. #ifdef CONFIG_NUMA
  2461. /* The value user specified ....changed by config */
  2462. static int user_zonelist_order = ZONELIST_ORDER_DEFAULT;
  2463. /* string for sysctl */
  2464. #define NUMA_ZONELIST_ORDER_LEN 16
  2465. char numa_zonelist_order[16] = "default";
  2466. /*
  2467. * interface for configure zonelist ordering.
  2468. * command line option "numa_zonelist_order"
  2469. * = "[dD]efault - default, automatic configuration.
  2470. * = "[nN]ode - order by node locality, then by zone within node
  2471. * = "[zZ]one - order by zone, then by locality within zone
  2472. */
  2473. static int __parse_numa_zonelist_order(char *s)
  2474. {
  2475. if (*s == 'd' || *s == 'D') {
  2476. user_zonelist_order = ZONELIST_ORDER_DEFAULT;
  2477. } else if (*s == 'n' || *s == 'N') {
  2478. user_zonelist_order = ZONELIST_ORDER_NODE;
  2479. } else if (*s == 'z' || *s == 'Z') {
  2480. user_zonelist_order = ZONELIST_ORDER_ZONE;
  2481. } else {
  2482. printk(KERN_WARNING
  2483. "Ignoring invalid numa_zonelist_order value: "
  2484. "%s\n", s);
  2485. return -EINVAL;
  2486. }
  2487. return 0;
  2488. }
  2489. static __init int setup_numa_zonelist_order(char *s)
  2490. {
  2491. int ret;
  2492. if (!s)
  2493. return 0;
  2494. ret = __parse_numa_zonelist_order(s);
  2495. if (ret == 0)
  2496. strlcpy(numa_zonelist_order, s, NUMA_ZONELIST_ORDER_LEN);
  2497. return ret;
  2498. }
  2499. early_param("numa_zonelist_order", setup_numa_zonelist_order);
  2500. /*
  2501. * sysctl handler for numa_zonelist_order
  2502. */
  2503. int numa_zonelist_order_handler(ctl_table *table, int write,
  2504. void __user *buffer, size_t *length,
  2505. loff_t *ppos)
  2506. {
  2507. char saved_string[NUMA_ZONELIST_ORDER_LEN];
  2508. int ret;
  2509. static DEFINE_MUTEX(zl_order_mutex);
  2510. mutex_lock(&zl_order_mutex);
  2511. if (write)
  2512. strcpy(saved_string, (char*)table->data);
  2513. ret = proc_dostring(table, write, buffer, length, ppos);
  2514. if (ret)
  2515. goto out;
  2516. if (write) {
  2517. int oldval = user_zonelist_order;
  2518. if (__parse_numa_zonelist_order((char*)table->data)) {
  2519. /*
  2520. * bogus value. restore saved string
  2521. */
  2522. strncpy((char*)table->data, saved_string,
  2523. NUMA_ZONELIST_ORDER_LEN);
  2524. user_zonelist_order = oldval;
  2525. } else if (oldval != user_zonelist_order) {
  2526. mutex_lock(&zonelists_mutex);
  2527. build_all_zonelists(NULL);
  2528. mutex_unlock(&zonelists_mutex);
  2529. }
  2530. }
  2531. out:
  2532. mutex_unlock(&zl_order_mutex);
  2533. return ret;
  2534. }
  2535. #define MAX_NODE_LOAD (nr_online_nodes)
  2536. static int node_load[MAX_NUMNODES];
  2537. /**
  2538. * find_next_best_node - find the next node that should appear in a given node's fallback list
  2539. * @node: node whose fallback list we're appending
  2540. * @used_node_mask: nodemask_t of already used nodes
  2541. *
  2542. * We use a number of factors to determine which is the next node that should
  2543. * appear on a given node's fallback list. The node should not have appeared
  2544. * already in @node's fallback list, and it should be the next closest node
  2545. * according to the distance array (which contains arbitrary distance values
  2546. * from each node to each node in the system), and should also prefer nodes
  2547. * with no CPUs, since presumably they'll have very little allocation pressure
  2548. * on them otherwise.
  2549. * It returns -1 if no node is found.
  2550. */
  2551. static int find_next_best_node(int node, nodemask_t *used_node_mask)
  2552. {
  2553. int n, val;
  2554. int min_val = INT_MAX;
  2555. int best_node = -1;
  2556. const struct cpumask *tmp = cpumask_of_node(0);
  2557. /* Use the local node if we haven't already */
  2558. if (!node_isset(node, *used_node_mask)) {
  2559. node_set(node, *used_node_mask);
  2560. return node;
  2561. }
  2562. for_each_node_state(n, N_HIGH_MEMORY) {
  2563. /* Don't want a node to appear more than once */
  2564. if (node_isset(n, *used_node_mask))
  2565. continue;
  2566. /* Use the distance array to find the distance */
  2567. val = node_distance(node, n);
  2568. /* Penalize nodes under us ("prefer the next node") */
  2569. val += (n < node);
  2570. /* Give preference to headless and unused nodes */
  2571. tmp = cpumask_of_node(n);
  2572. if (!cpumask_empty(tmp))
  2573. val += PENALTY_FOR_NODE_WITH_CPUS;
  2574. /* Slight preference for less loaded node */
  2575. val *= (MAX_NODE_LOAD*MAX_NUMNODES);
  2576. val += node_load[n];
  2577. if (val < min_val) {
  2578. min_val = val;
  2579. best_node = n;
  2580. }
  2581. }
  2582. if (best_node >= 0)
  2583. node_set(best_node, *used_node_mask);
  2584. return best_node;
  2585. }
  2586. /*
  2587. * Build zonelists ordered by node and zones within node.
  2588. * This results in maximum locality--normal zone overflows into local
  2589. * DMA zone, if any--but risks exhausting DMA zone.
  2590. */
  2591. static void build_zonelists_in_node_order(pg_data_t *pgdat, int node)
  2592. {
  2593. int j;
  2594. struct zonelist *zonelist;
  2595. zonelist = &pgdat->node_zonelists[0];
  2596. for (j = 0; zonelist->_zonerefs[j].zone != NULL; j++)
  2597. ;
  2598. j = build_zonelists_node(NODE_DATA(node), zonelist, j,
  2599. MAX_NR_ZONES - 1);
  2600. zonelist->_zonerefs[j].zone = NULL;
  2601. zonelist->_zonerefs[j].zone_idx = 0;
  2602. }
  2603. /*
  2604. * Build gfp_thisnode zonelists
  2605. */
  2606. static void build_thisnode_zonelists(pg_data_t *pgdat)
  2607. {
  2608. int j;
  2609. struct zonelist *zonelist;
  2610. zonelist = &pgdat->node_zonelists[1];
  2611. j = build_zonelists_node(pgdat, zonelist, 0, MAX_NR_ZONES - 1);
  2612. zonelist->_zonerefs[j].zone = NULL;
  2613. zonelist->_zonerefs[j].zone_idx = 0;
  2614. }
  2615. /*
  2616. * Build zonelists ordered by zone and nodes within zones.
  2617. * This results in conserving DMA zone[s] until all Normal memory is
  2618. * exhausted, but results in overflowing to remote node while memory
  2619. * may still exist in local DMA zone.
  2620. */
  2621. static int node_order[MAX_NUMNODES];
  2622. static void build_zonelists_in_zone_order(pg_data_t *pgdat, int nr_nodes)
  2623. {
  2624. int pos, j, node;
  2625. int zone_type; /* needs to be signed */
  2626. struct zone *z;
  2627. struct zonelist *zonelist;
  2628. zonelist = &pgdat->node_zonelists[0];
  2629. pos = 0;
  2630. for (zone_type = MAX_NR_ZONES - 1; zone_type >= 0; zone_type--) {
  2631. for (j = 0; j < nr_nodes; j++) {
  2632. node = node_order[j];
  2633. z = &NODE_DATA(node)->node_zones[zone_type];
  2634. if (populated_zone(z)) {
  2635. zoneref_set_zone(z,
  2636. &zonelist->_zonerefs[pos++]);
  2637. check_highest_zone(zone_type);
  2638. }
  2639. }
  2640. }
  2641. zonelist->_zonerefs[pos].zone = NULL;
  2642. zonelist->_zonerefs[pos].zone_idx = 0;
  2643. }
  2644. static int default_zonelist_order(void)
  2645. {
  2646. int nid, zone_type;
  2647. unsigned long low_kmem_size,total_size;
  2648. struct zone *z;
  2649. int average_size;
  2650. /*
  2651. * ZONE_DMA and ZONE_DMA32 can be very small area in the system.
  2652. * If they are really small and used heavily, the system can fall
  2653. * into OOM very easily.
  2654. * This function detect ZONE_DMA/DMA32 size and configures zone order.
  2655. */
  2656. /* Is there ZONE_NORMAL ? (ex. ppc has only DMA zone..) */
  2657. low_kmem_size = 0;
  2658. total_size = 0;
  2659. for_each_online_node(nid) {
  2660. for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
  2661. z = &NODE_DATA(nid)->node_zones[zone_type];
  2662. if (populated_zone(z)) {
  2663. if (zone_type < ZONE_NORMAL)
  2664. low_kmem_size += z->present_pages;
  2665. total_size += z->present_pages;
  2666. } else if (zone_type == ZONE_NORMAL) {
  2667. /*
  2668. * If any node has only lowmem, then node order
  2669. * is preferred to allow kernel allocations
  2670. * locally; otherwise, they can easily infringe
  2671. * on other nodes when there is an abundance of
  2672. * lowmem available to allocate from.
  2673. */
  2674. return ZONELIST_ORDER_NODE;
  2675. }
  2676. }
  2677. }
  2678. if (!low_kmem_size || /* there are no DMA area. */
  2679. low_kmem_size > total_size/2) /* DMA/DMA32 is big. */
  2680. return ZONELIST_ORDER_NODE;
  2681. /*
  2682. * look into each node's config.
  2683. * If there is a node whose DMA/DMA32 memory is very big area on
  2684. * local memory, NODE_ORDER may be suitable.
  2685. */
  2686. average_size = total_size /
  2687. (nodes_weight(node_states[N_HIGH_MEMORY]) + 1);
  2688. for_each_online_node(nid) {
  2689. low_kmem_size = 0;
  2690. total_size = 0;
  2691. for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
  2692. z = &NODE_DATA(nid)->node_zones[zone_type];
  2693. if (populated_zone(z)) {
  2694. if (zone_type < ZONE_NORMAL)
  2695. low_kmem_size += z->present_pages;
  2696. total_size += z->present_pages;
  2697. }
  2698. }
  2699. if (low_kmem_size &&
  2700. total_size > average_size && /* ignore small node */
  2701. low_kmem_size > total_size * 70/100)
  2702. return ZONELIST_ORDER_NODE;
  2703. }
  2704. return ZONELIST_ORDER_ZONE;
  2705. }
  2706. static void set_zonelist_order(void)
  2707. {
  2708. if (user_zonelist_order == ZONELIST_ORDER_DEFAULT)
  2709. current_zonelist_order = default_zonelist_order();
  2710. else
  2711. current_zonelist_order = user_zonelist_order;
  2712. }
  2713. static void build_zonelists(pg_data_t *pgdat)
  2714. {
  2715. int j, node, load;
  2716. enum zone_type i;
  2717. nodemask_t used_mask;
  2718. int local_node, prev_node;
  2719. struct zonelist *zonelist;
  2720. int order = current_zonelist_order;
  2721. /* initialize zonelists */
  2722. for (i = 0; i < MAX_ZONELISTS; i++) {
  2723. zonelist = pgdat->node_zonelists + i;
  2724. zonelist->_zonerefs[0].zone = NULL;
  2725. zonelist->_zonerefs[0].zone_idx = 0;
  2726. }
  2727. /* NUMA-aware ordering of nodes */
  2728. local_node = pgdat->node_id;
  2729. load = nr_online_nodes;
  2730. prev_node = local_node;
  2731. nodes_clear(used_mask);
  2732. memset(node_order, 0, sizeof(node_order));
  2733. j = 0;
  2734. while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
  2735. int distance = node_distance(local_node, node);
  2736. /*
  2737. * If another node is sufficiently far away then it is better
  2738. * to reclaim pages in a zone before going off node.
  2739. */
  2740. if (distance > RECLAIM_DISTANCE)
  2741. zone_reclaim_mode = 1;
  2742. /*
  2743. * We don't want to pressure a particular node.
  2744. * So adding penalty to the first node in same
  2745. * distance group to make it round-robin.
  2746. */
  2747. if (distance != node_distance(local_node, prev_node))
  2748. node_load[node] = load;
  2749. prev_node = node;
  2750. load--;
  2751. if (order == ZONELIST_ORDER_NODE)
  2752. build_zonelists_in_node_order(pgdat, node);
  2753. else
  2754. node_order[j++] = node; /* remember order */
  2755. }
  2756. if (order == ZONELIST_ORDER_ZONE) {
  2757. /* calculate node order -- i.e., DMA last! */
  2758. build_zonelists_in_zone_order(pgdat, j);
  2759. }
  2760. build_thisnode_zonelists(pgdat);
  2761. }
  2762. /* Construct the zonelist performance cache - see further mmzone.h */
  2763. static void build_zonelist_cache(pg_data_t *pgdat)
  2764. {
  2765. struct zonelist *zonelist;
  2766. struct zonelist_cache *zlc;
  2767. struct zoneref *z;
  2768. zonelist = &pgdat->node_zonelists[0];
  2769. zonelist->zlcache_ptr = zlc = &zonelist->zlcache;
  2770. bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
  2771. for (z = zonelist->_zonerefs; z->zone; z++)
  2772. zlc->z_to_n[z - zonelist->_zonerefs] = zonelist_node_idx(z);
  2773. }
  2774. #ifdef CONFIG_HAVE_MEMORYLESS_NODES
  2775. /*
  2776. * Return node id of node used for "local" allocations.
  2777. * I.e., first node id of first zone in arg node's generic zonelist.
  2778. * Used for initializing percpu 'numa_mem', which is used primarily
  2779. * for kernel allocations, so use GFP_KERNEL flags to locate zonelist.
  2780. */
  2781. int local_memory_node(int node)
  2782. {
  2783. struct zone *zone;
  2784. (void)first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
  2785. gfp_zone(GFP_KERNEL),
  2786. NULL,
  2787. &zone);
  2788. return zone->node;
  2789. }
  2790. #endif
  2791. #else /* CONFIG_NUMA */
  2792. static void set_zonelist_order(void)
  2793. {
  2794. current_zonelist_order = ZONELIST_ORDER_ZONE;
  2795. }
  2796. static void build_zonelists(pg_data_t *pgdat)
  2797. {
  2798. int node, local_node;
  2799. enum zone_type j;
  2800. struct zonelist *zonelist;
  2801. local_node = pgdat->node_id;
  2802. zonelist = &pgdat->node_zonelists[0];
  2803. j = build_zonelists_node(pgdat, zonelist, 0, MAX_NR_ZONES - 1);
  2804. /*
  2805. * Now we build the zonelist so that it contains the zones
  2806. * of all the other nodes.
  2807. * We don't want to pressure a particular node, so when
  2808. * building the zones for node N, we make sure that the
  2809. * zones coming right after the local ones are those from
  2810. * node N+1 (modulo N)
  2811. */
  2812. for (node = local_node + 1; node < MAX_NUMNODES; node++) {
  2813. if (!node_online(node))
  2814. continue;
  2815. j = build_zonelists_node(NODE_DATA(node), zonelist, j,
  2816. MAX_NR_ZONES - 1);
  2817. }
  2818. for (node = 0; node < local_node; node++) {
  2819. if (!node_online(node))
  2820. continue;
  2821. j = build_zonelists_node(NODE_DATA(node), zonelist, j,
  2822. MAX_NR_ZONES - 1);
  2823. }
  2824. zonelist->_zonerefs[j].zone = NULL;
  2825. zonelist->_zonerefs[j].zone_idx = 0;
  2826. }
  2827. /* non-NUMA variant of zonelist performance cache - just NULL zlcache_ptr */
  2828. static void build_zonelist_cache(pg_data_t *pgdat)
  2829. {
  2830. pgdat->node_zonelists[0].zlcache_ptr = NULL;
  2831. }
  2832. #endif /* CONFIG_NUMA */
  2833. /*
  2834. * Boot pageset table. One per cpu which is going to be used for all
  2835. * zones and all nodes. The parameters will be set in such a way
  2836. * that an item put on a list will immediately be handed over to
  2837. * the buddy list. This is safe since pageset manipulation is done
  2838. * with interrupts disabled.
  2839. *
  2840. * The boot_pagesets must be kept even after bootup is complete for
  2841. * unused processors and/or zones. They do play a role for bootstrapping
  2842. * hotplugged processors.
  2843. *
  2844. * zoneinfo_show() and maybe other functions do
  2845. * not check if the processor is online before following the pageset pointer.
  2846. * Other parts of the kernel may not check if the zone is available.
  2847. */
  2848. static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
  2849. static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
  2850. static void setup_zone_pageset(struct zone *zone);
  2851. /*
  2852. * Global mutex to protect against size modification of zonelists
  2853. * as well as to serialize pageset setup for the new populated zone.
  2854. */
  2855. DEFINE_MUTEX(zonelists_mutex);
  2856. /* return values int ....just for stop_machine() */
  2857. static __init_refok int __build_all_zonelists(void *data)
  2858. {
  2859. int nid;
  2860. int cpu;
  2861. #ifdef CONFIG_NUMA
  2862. memset(node_load, 0, sizeof(node_load));
  2863. #endif
  2864. for_each_online_node(nid) {
  2865. pg_data_t *pgdat = NODE_DATA(nid);
  2866. build_zonelists(pgdat);
  2867. build_zonelist_cache(pgdat);
  2868. }
  2869. /*
  2870. * Initialize the boot_pagesets that are going to be used
  2871. * for bootstrapping processors. The real pagesets for
  2872. * each zone will be allocated later when the per cpu
  2873. * allocator is available.
  2874. *
  2875. * boot_pagesets are used also for bootstrapping offline
  2876. * cpus if the system is already booted because the pagesets
  2877. * are needed to initialize allocators on a specific cpu too.
  2878. * F.e. the percpu allocator needs the page allocator which
  2879. * needs the percpu allocator in order to allocate its pagesets
  2880. * (a chicken-egg dilemma).
  2881. */
  2882. for_each_possible_cpu(cpu) {
  2883. setup_pageset(&per_cpu(boot_pageset, cpu), 0);
  2884. #ifdef CONFIG_HAVE_MEMORYLESS_NODES
  2885. /*
  2886. * We now know the "local memory node" for each node--
  2887. * i.e., the node of the first zone in the generic zonelist.
  2888. * Set up numa_mem percpu variable for on-line cpus. During
  2889. * boot, only the boot cpu should be on-line; we'll init the
  2890. * secondary cpus' numa_mem as they come on-line. During
  2891. * node/memory hotplug, we'll fixup all on-line cpus.
  2892. */
  2893. if (cpu_online(cpu))
  2894. set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu)));
  2895. #endif
  2896. }
  2897. return 0;
  2898. }
  2899. /*
  2900. * Called with zonelists_mutex held always
  2901. * unless system_state == SYSTEM_BOOTING.
  2902. */
  2903. void __ref build_all_zonelists(void *data)
  2904. {
  2905. set_zonelist_order();
  2906. if (system_state == SYSTEM_BOOTING) {
  2907. __build_all_zonelists(NULL);
  2908. mminit_verify_zonelist();
  2909. cpuset_init_current_mems_allowed();
  2910. } else {
  2911. /* we have to stop all cpus to guarantee there is no user
  2912. of zonelist */
  2913. #ifdef CONFIG_MEMORY_HOTPLUG
  2914. if (data)
  2915. setup_zone_pageset((struct zone *)data);
  2916. #endif
  2917. stop_machine(__build_all_zonelists, NULL, NULL);
  2918. /* cpuset refresh routine should be here */
  2919. }
  2920. vm_total_pages = nr_free_pagecache_pages();
  2921. /*
  2922. * Disable grouping by mobility if the number of pages in the
  2923. * system is too low to allow the mechanism to work. It would be
  2924. * more accurate, but expensive to check per-zone. This check is
  2925. * made on memory-hotadd so a system can start with mobility
  2926. * disabled and enable it later
  2927. */
  2928. if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
  2929. page_group_by_mobility_disabled = 1;
  2930. else
  2931. page_group_by_mobility_disabled = 0;
  2932. printk("Built %i zonelists in %s order, mobility grouping %s. "
  2933. "Total pages: %ld\n",
  2934. nr_online_nodes,
  2935. zonelist_order_name[current_zonelist_order],
  2936. page_group_by_mobility_disabled ? "off" : "on",
  2937. vm_total_pages);
  2938. #ifdef CONFIG_NUMA
  2939. printk("Policy zone: %s\n", zone_names[policy_zone]);
  2940. #endif
  2941. }
  2942. /*
  2943. * Helper functions to size the waitqueue hash table.
  2944. * Essentially these want to choose hash table sizes sufficiently
  2945. * large so that collisions trying to wait on pages are rare.
  2946. * But in fact, the number of active page waitqueues on typical
  2947. * systems is ridiculously low, less than 200. So this is even
  2948. * conservative, even though it seems large.
  2949. *
  2950. * The constant PAGES_PER_WAITQUEUE specifies the ratio of pages to
  2951. * waitqueues, i.e. the size of the waitq table given the number of pages.
  2952. */
  2953. #define PAGES_PER_WAITQUEUE 256
  2954. #ifndef CONFIG_MEMORY_HOTPLUG
  2955. static inline unsigned long wait_table_hash_nr_entries(unsigned long pages)
  2956. {
  2957. unsigned long size = 1;
  2958. pages /= PAGES_PER_WAITQUEUE;
  2959. while (size < pages)
  2960. size <<= 1;
  2961. /*
  2962. * Once we have dozens or even hundreds of threads sleeping
  2963. * on IO we've got bigger problems than wait queue collision.
  2964. * Limit the size of the wait table to a reasonable size.
  2965. */
  2966. size = min(size, 4096UL);
  2967. return max(size, 4UL);
  2968. }
  2969. #else
  2970. /*
  2971. * A zone's size might be changed by hot-add, so it is not possible to determine
  2972. * a suitable size for its wait_table. So we use the maximum size now.
  2973. *
  2974. * The max wait table size = 4096 x sizeof(wait_queue_head_t). ie:
  2975. *
  2976. * i386 (preemption config) : 4096 x 16 = 64Kbyte.
  2977. * ia64, x86-64 (no preemption): 4096 x 20 = 80Kbyte.
  2978. * ia64, x86-64 (preemption) : 4096 x 24 = 96Kbyte.
  2979. *
  2980. * The maximum entries are prepared when a zone's memory is (512K + 256) pages
  2981. * or more by the traditional way. (See above). It equals:
  2982. *
  2983. * i386, x86-64, powerpc(4K page size) : = ( 2G + 1M)byte.
  2984. * ia64(16K page size) : = ( 8G + 4M)byte.
  2985. * powerpc (64K page size) : = (32G +16M)byte.
  2986. */
  2987. static inline unsigned long wait_table_hash_nr_entries(unsigned long pages)
  2988. {
  2989. return 4096UL;
  2990. }
  2991. #endif
  2992. /*
  2993. * This is an integer logarithm so that shifts can be used later
  2994. * to extract the more random high bits from the multiplicative
  2995. * hash function before the remainder is taken.
  2996. */
  2997. static inline unsigned long wait_table_bits(unsigned long size)
  2998. {
  2999. return ffz(~size);
  3000. }
  3001. #define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1))
  3002. /*
  3003. * Check if a pageblock contains reserved pages
  3004. */
  3005. static int pageblock_is_reserved(unsigned long start_pfn, unsigned long end_pfn)
  3006. {
  3007. unsigned long pfn;
  3008. for (pfn = start_pfn; pfn < end_pfn; pfn++) {
  3009. if (!pfn_valid_within(pfn) || PageReserved(pfn_to_page(pfn)))
  3010. return 1;
  3011. }
  3012. return 0;
  3013. }
  3014. /*
  3015. * Mark a number of pageblocks as MIGRATE_RESERVE. The number
  3016. * of blocks reserved is based on min_wmark_pages(zone). The memory within
  3017. * the reserve will tend to store contiguous free pages. Setting min_free_kbytes
  3018. * higher will lead to a bigger reserve which will get freed as contiguous
  3019. * blocks as reclaim kicks in
  3020. */
  3021. static void setup_zone_migrate_reserve(struct zone *zone)
  3022. {
  3023. unsigned long start_pfn, pfn, end_pfn, block_end_pfn;
  3024. struct page *page;
  3025. unsigned long block_migratetype;
  3026. int reserve;
  3027. /*
  3028. * Get the start pfn, end pfn and the number of blocks to reserve
  3029. * We have to be careful to be aligned to pageblock_nr_pages to
  3030. * make sure that we always check pfn_valid for the first page in
  3031. * the block.
  3032. */
  3033. start_pfn = zone->zone_start_pfn;
  3034. end_pfn = start_pfn + zone->spanned_pages;
  3035. start_pfn = roundup(start_pfn, pageblock_nr_pages);
  3036. reserve = roundup(min_wmark_pages(zone), pageblock_nr_pages) >>
  3037. pageblock_order;
  3038. /*
  3039. * Reserve blocks are generally in place to help high-order atomic
  3040. * allocations that are short-lived. A min_free_kbytes value that
  3041. * would result in more than 2 reserve blocks for atomic allocations
  3042. * is assumed to be in place to help anti-fragmentation for the
  3043. * future allocation of hugepages at runtime.
  3044. */
  3045. reserve = min(2, reserve);
  3046. for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
  3047. if (!pfn_valid(pfn))
  3048. continue;
  3049. page = pfn_to_page(pfn);
  3050. /* Watch out for overlapping nodes */
  3051. if (page_to_nid(page) != zone_to_nid(zone))
  3052. continue;
  3053. block_migratetype = get_pageblock_migratetype(page);
  3054. /* Only test what is necessary when the reserves are not met */
  3055. if (reserve > 0) {
  3056. /*
  3057. * Blocks with reserved pages will never free, skip
  3058. * them.
  3059. */
  3060. block_end_pfn = min(pfn + pageblock_nr_pages, end_pfn);
  3061. if (pageblock_is_reserved(pfn, block_end_pfn))
  3062. continue;
  3063. /* If this block is reserved, account for it */
  3064. if (block_migratetype == MIGRATE_RESERVE) {
  3065. reserve--;
  3066. continue;
  3067. }
  3068. /* Suitable for reserving if this block is movable */
  3069. if (block_migratetype == MIGRATE_MOVABLE) {
  3070. set_pageblock_migratetype(page,
  3071. MIGRATE_RESERVE);
  3072. move_freepages_block(zone, page,
  3073. MIGRATE_RESERVE);
  3074. reserve--;
  3075. continue;
  3076. }
  3077. }
  3078. /*
  3079. * If the reserve is met and this is a previous reserved block,
  3080. * take it back
  3081. */
  3082. if (block_migratetype == MIGRATE_RESERVE) {
  3083. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  3084. move_freepages_block(zone, page, MIGRATE_MOVABLE);
  3085. }
  3086. }
  3087. }
  3088. /*
  3089. * Initially all pages are reserved - free ones are freed
  3090. * up by free_all_bootmem() once the early boot process is
  3091. * done. Non-atomic initialization, single-pass.
  3092. */
  3093. void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
  3094. unsigned long start_pfn, enum memmap_context context)
  3095. {
  3096. struct page *page;
  3097. unsigned long end_pfn = start_pfn + size;
  3098. unsigned long pfn;
  3099. struct zone *z;
  3100. if (highest_memmap_pfn < end_pfn - 1)
  3101. highest_memmap_pfn = end_pfn - 1;
  3102. z = &NODE_DATA(nid)->node_zones[zone];
  3103. for (pfn = start_pfn; pfn < end_pfn; pfn++) {
  3104. /*
  3105. * There can be holes in boot-time mem_map[]s
  3106. * handed to this function. They do not
  3107. * exist on hotplugged memory.
  3108. */
  3109. if (context == MEMMAP_EARLY) {
  3110. if (!early_pfn_valid(pfn))
  3111. continue;
  3112. if (!early_pfn_in_nid(pfn, nid))
  3113. continue;
  3114. }
  3115. page = pfn_to_page(pfn);
  3116. set_page_links(page, zone, nid, pfn);
  3117. mminit_verify_page_links(page, zone, nid, pfn);
  3118. init_page_count(page);
  3119. reset_page_mapcount(page);
  3120. SetPageReserved(page);
  3121. /*
  3122. * Mark the block movable so that blocks are reserved for
  3123. * movable at startup. This will force kernel allocations
  3124. * to reserve their blocks rather than leaking throughout
  3125. * the address space during boot when many long-lived
  3126. * kernel allocations are made. Later some blocks near
  3127. * the start are marked MIGRATE_RESERVE by
  3128. * setup_zone_migrate_reserve()
  3129. *
  3130. * bitmap is created for zone's valid pfn range. but memmap
  3131. * can be created for invalid pages (for alignment)
  3132. * check here not to call set_pageblock_migratetype() against
  3133. * pfn out of zone.
  3134. */
  3135. if ((z->zone_start_pfn <= pfn)
  3136. && (pfn < z->zone_start_pfn + z->spanned_pages)
  3137. && !(pfn & (pageblock_nr_pages - 1)))
  3138. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  3139. INIT_LIST_HEAD(&page->lru);
  3140. #ifdef WANT_PAGE_VIRTUAL
  3141. /* The shift won't overflow because ZONE_NORMAL is below 4G. */
  3142. if (!is_highmem_idx(zone))
  3143. set_page_address(page, __va(pfn << PAGE_SHIFT));
  3144. #endif
  3145. }
  3146. }
  3147. static void __meminit zone_init_free_lists(struct zone *zone)
  3148. {
  3149. int order, t;
  3150. for_each_migratetype_order(order, t) {
  3151. INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
  3152. zone->free_area[order].nr_free = 0;
  3153. }
  3154. }
  3155. #ifndef __HAVE_ARCH_MEMMAP_INIT
  3156. #define memmap_init(size, nid, zone, start_pfn) \
  3157. memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY)
  3158. #endif
  3159. static int zone_batchsize(struct zone *zone)
  3160. {
  3161. #ifdef CONFIG_MMU
  3162. int batch;
  3163. /*
  3164. * The per-cpu-pages pools are set to around 1000th of the
  3165. * size of the zone. But no more than 1/2 of a meg.
  3166. *
  3167. * OK, so we don't know how big the cache is. So guess.
  3168. */
  3169. batch = zone->present_pages / 1024;
  3170. if (batch * PAGE_SIZE > 512 * 1024)
  3171. batch = (512 * 1024) / PAGE_SIZE;
  3172. batch /= 4; /* We effectively *= 4 below */
  3173. if (batch < 1)
  3174. batch = 1;
  3175. /*
  3176. * Clamp the batch to a 2^n - 1 value. Having a power
  3177. * of 2 value was found to be more likely to have
  3178. * suboptimal cache aliasing properties in some cases.
  3179. *
  3180. * For example if 2 tasks are alternately allocating
  3181. * batches of pages, one task can end up with a lot
  3182. * of pages of one half of the possible page colors
  3183. * and the other with pages of the other colors.
  3184. */
  3185. batch = rounddown_pow_of_two(batch + batch/2) - 1;
  3186. return batch;
  3187. #else
  3188. /* The deferral and batching of frees should be suppressed under NOMMU
  3189. * conditions.
  3190. *
  3191. * The problem is that NOMMU needs to be able to allocate large chunks
  3192. * of contiguous memory as there's no hardware page translation to
  3193. * assemble apparent contiguous memory from discontiguous pages.
  3194. *
  3195. * Queueing large contiguous runs of pages for batching, however,
  3196. * causes the pages to actually be freed in smaller chunks. As there
  3197. * can be a significant delay between the individual batches being
  3198. * recycled, this leads to the once large chunks of space being
  3199. * fragmented and becoming unavailable for high-order allocations.
  3200. */
  3201. return 0;
  3202. #endif
  3203. }
  3204. static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
  3205. {
  3206. struct per_cpu_pages *pcp;
  3207. int migratetype;
  3208. memset(p, 0, sizeof(*p));
  3209. pcp = &p->pcp;
  3210. pcp->count = 0;
  3211. pcp->high = 6 * batch;
  3212. pcp->batch = max(1UL, 1 * batch);
  3213. for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++)
  3214. INIT_LIST_HEAD(&pcp->lists[migratetype]);
  3215. }
  3216. /*
  3217. * setup_pagelist_highmark() sets the high water mark for hot per_cpu_pagelist
  3218. * to the value high for the pageset p.
  3219. */
  3220. static void setup_pagelist_highmark(struct per_cpu_pageset *p,
  3221. unsigned long high)
  3222. {
  3223. struct per_cpu_pages *pcp;
  3224. pcp = &p->pcp;
  3225. pcp->high = high;
  3226. pcp->batch = max(1UL, high/4);
  3227. if ((high/4) > (PAGE_SHIFT * 8))
  3228. pcp->batch = PAGE_SHIFT * 8;
  3229. }
  3230. static void setup_zone_pageset(struct zone *zone)
  3231. {
  3232. int cpu;
  3233. zone->pageset = alloc_percpu(struct per_cpu_pageset);
  3234. for_each_possible_cpu(cpu) {
  3235. struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);
  3236. setup_pageset(pcp, zone_batchsize(zone));
  3237. if (percpu_pagelist_fraction)
  3238. setup_pagelist_highmark(pcp,
  3239. (zone->present_pages /
  3240. percpu_pagelist_fraction));
  3241. }
  3242. }
  3243. /*
  3244. * Allocate per cpu pagesets and initialize them.
  3245. * Before this call only boot pagesets were available.
  3246. */
  3247. void __init setup_per_cpu_pageset(void)
  3248. {
  3249. struct zone *zone;
  3250. for_each_populated_zone(zone)
  3251. setup_zone_pageset(zone);
  3252. }
  3253. static noinline __init_refok
  3254. int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages)
  3255. {
  3256. int i;
  3257. struct pglist_data *pgdat = zone->zone_pgdat;
  3258. size_t alloc_size;
  3259. /*
  3260. * The per-page waitqueue mechanism uses hashed waitqueues
  3261. * per zone.
  3262. */
  3263. zone->wait_table_hash_nr_entries =
  3264. wait_table_hash_nr_entries(zone_size_pages);
  3265. zone->wait_table_bits =
  3266. wait_table_bits(zone->wait_table_hash_nr_entries);
  3267. alloc_size = zone->wait_table_hash_nr_entries
  3268. * sizeof(wait_queue_head_t);
  3269. if (!slab_is_available()) {
  3270. zone->wait_table = (wait_queue_head_t *)
  3271. alloc_bootmem_node_nopanic(pgdat, alloc_size);
  3272. } else {
  3273. /*
  3274. * This case means that a zone whose size was 0 gets new memory
  3275. * via memory hot-add.
  3276. * But it may be the case that a new node was hot-added. In
  3277. * this case vmalloc() will not be able to use this new node's
  3278. * memory - this wait_table must be initialized to use this new
  3279. * node itself as well.
  3280. * To use this new node's memory, further consideration will be
  3281. * necessary.
  3282. */
  3283. zone->wait_table = vmalloc(alloc_size);
  3284. }
  3285. if (!zone->wait_table)
  3286. return -ENOMEM;
  3287. for(i = 0; i < zone->wait_table_hash_nr_entries; ++i)
  3288. init_waitqueue_head(zone->wait_table + i);
  3289. return 0;
  3290. }
  3291. static int __zone_pcp_update(void *data)
  3292. {
  3293. struct zone *zone = data;
  3294. int cpu;
  3295. unsigned long batch = zone_batchsize(zone), flags;
  3296. for_each_possible_cpu(cpu) {
  3297. struct per_cpu_pageset *pset;
  3298. struct per_cpu_pages *pcp;
  3299. pset = per_cpu_ptr(zone->pageset, cpu);
  3300. pcp = &pset->pcp;
  3301. local_irq_save(flags);
  3302. free_pcppages_bulk(zone, pcp->count, pcp);
  3303. setup_pageset(pset, batch);
  3304. local_irq_restore(flags);
  3305. }
  3306. return 0;
  3307. }
  3308. void zone_pcp_update(struct zone *zone)
  3309. {
  3310. stop_machine(__zone_pcp_update, zone, NULL);
  3311. }
  3312. static __meminit void zone_pcp_init(struct zone *zone)
  3313. {
  3314. /*
  3315. * per cpu subsystem is not up at this point. The following code
  3316. * relies on the ability of the linker to provide the
  3317. * offset of a (static) per cpu variable into the per cpu area.
  3318. */
  3319. zone->pageset = &boot_pageset;
  3320. if (zone->present_pages)
  3321. printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n",
  3322. zone->name, zone->present_pages,
  3323. zone_batchsize(zone));
  3324. }
  3325. __meminit int init_currently_empty_zone(struct zone *zone,
  3326. unsigned long zone_start_pfn,
  3327. unsigned long size,
  3328. enum memmap_context context)
  3329. {
  3330. struct pglist_data *pgdat = zone->zone_pgdat;
  3331. int ret;
  3332. ret = zone_wait_table_init(zone, size);
  3333. if (ret)
  3334. return ret;
  3335. pgdat->nr_zones = zone_idx(zone) + 1;
  3336. zone->zone_start_pfn = zone_start_pfn;
  3337. mminit_dprintk(MMINIT_TRACE, "memmap_init",
  3338. "Initialising map node %d zone %lu pfns %lu -> %lu\n",
  3339. pgdat->node_id,
  3340. (unsigned long)zone_idx(zone),
  3341. zone_start_pfn, (zone_start_pfn + size));
  3342. zone_init_free_lists(zone);
  3343. return 0;
  3344. }
  3345. #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
  3346. #ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
  3347. /*
  3348. * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
  3349. * Architectures may implement their own version but if add_active_range()
  3350. * was used and there are no special requirements, this is a convenient
  3351. * alternative
  3352. */
  3353. int __meminit __early_pfn_to_nid(unsigned long pfn)
  3354. {
  3355. unsigned long start_pfn, end_pfn;
  3356. int i, nid;
  3357. for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
  3358. if (start_pfn <= pfn && pfn < end_pfn)
  3359. return nid;
  3360. /* This is a memory hole */
  3361. return -1;
  3362. }
  3363. #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
  3364. int __meminit early_pfn_to_nid(unsigned long pfn)
  3365. {
  3366. int nid;
  3367. nid = __early_pfn_to_nid(pfn);
  3368. if (nid >= 0)
  3369. return nid;
  3370. /* just returns 0 */
  3371. return 0;
  3372. }
  3373. #ifdef CONFIG_NODES_SPAN_OTHER_NODES
  3374. bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
  3375. {
  3376. int nid;
  3377. nid = __early_pfn_to_nid(pfn);
  3378. if (nid >= 0 && nid != node)
  3379. return false;
  3380. return true;
  3381. }
  3382. #endif
  3383. /**
  3384. * free_bootmem_with_active_regions - Call free_bootmem_node for each active range
  3385. * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed.
  3386. * @max_low_pfn: The highest PFN that will be passed to free_bootmem_node
  3387. *
  3388. * If an architecture guarantees that all ranges registered with
  3389. * add_active_ranges() contain no holes and may be freed, this
  3390. * this function may be used instead of calling free_bootmem() manually.
  3391. */
  3392. void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn)
  3393. {
  3394. unsigned long start_pfn, end_pfn;
  3395. int i, this_nid;
  3396. for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) {
  3397. start_pfn = min(start_pfn, max_low_pfn);
  3398. end_pfn = min(end_pfn, max_low_pfn);
  3399. if (start_pfn < end_pfn)
  3400. free_bootmem_node(NODE_DATA(this_nid),
  3401. PFN_PHYS(start_pfn),
  3402. (end_pfn - start_pfn) << PAGE_SHIFT);
  3403. }
  3404. }
  3405. int __init add_from_early_node_map(struct range *range, int az,
  3406. int nr_range, int nid)
  3407. {
  3408. unsigned long start_pfn, end_pfn;
  3409. int i;
  3410. /* need to go over early_node_map to find out good range for node */
  3411. for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL)
  3412. nr_range = add_range(range, az, nr_range, start_pfn, end_pfn);
  3413. return nr_range;
  3414. }
  3415. /**
  3416. * sparse_memory_present_with_active_regions - Call memory_present for each active range
  3417. * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used.
  3418. *
  3419. * If an architecture guarantees that all ranges registered with
  3420. * add_active_ranges() contain no holes and may be freed, this
  3421. * function may be used instead of calling memory_present() manually.
  3422. */
  3423. void __init sparse_memory_present_with_active_regions(int nid)
  3424. {
  3425. unsigned long start_pfn, end_pfn;
  3426. int i, this_nid;
  3427. for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid)
  3428. memory_present(this_nid, start_pfn, end_pfn);
  3429. }
  3430. /**
  3431. * get_pfn_range_for_nid - Return the start and end page frames for a node
  3432. * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
  3433. * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
  3434. * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
  3435. *
  3436. * It returns the start and end page frame of a node based on information
  3437. * provided by an arch calling add_active_range(). If called for a node
  3438. * with no available memory, a warning is printed and the start and end
  3439. * PFNs will be 0.
  3440. */
  3441. void __meminit get_pfn_range_for_nid(unsigned int nid,
  3442. unsigned long *start_pfn, unsigned long *end_pfn)
  3443. {
  3444. unsigned long this_start_pfn, this_end_pfn;
  3445. int i;
  3446. *start_pfn = -1UL;
  3447. *end_pfn = 0;
  3448. for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) {
  3449. *start_pfn = min(*start_pfn, this_start_pfn);
  3450. *end_pfn = max(*end_pfn, this_end_pfn);
  3451. }
  3452. if (*start_pfn == -1UL)
  3453. *start_pfn = 0;
  3454. }
  3455. /*
  3456. * This finds a zone that can be used for ZONE_MOVABLE pages. The
  3457. * assumption is made that zones within a node are ordered in monotonic
  3458. * increasing memory addresses so that the "highest" populated zone is used
  3459. */
  3460. static void __init find_usable_zone_for_movable(void)
  3461. {
  3462. int zone_index;
  3463. for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
  3464. if (zone_index == ZONE_MOVABLE)
  3465. continue;
  3466. if (arch_zone_highest_possible_pfn[zone_index] >
  3467. arch_zone_lowest_possible_pfn[zone_index])
  3468. break;
  3469. }
  3470. VM_BUG_ON(zone_index == -1);
  3471. movable_zone = zone_index;
  3472. }
  3473. /*
  3474. * The zone ranges provided by the architecture do not include ZONE_MOVABLE
  3475. * because it is sized independent of architecture. Unlike the other zones,
  3476. * the starting point for ZONE_MOVABLE is not fixed. It may be different
  3477. * in each node depending on the size of each node and how evenly kernelcore
  3478. * is distributed. This helper function adjusts the zone ranges
  3479. * provided by the architecture for a given node by using the end of the
  3480. * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
  3481. * zones within a node are in order of monotonic increases memory addresses
  3482. */
  3483. static void __meminit adjust_zone_range_for_zone_movable(int nid,
  3484. unsigned long zone_type,
  3485. unsigned long node_start_pfn,
  3486. unsigned long node_end_pfn,
  3487. unsigned long *zone_start_pfn,
  3488. unsigned long *zone_end_pfn)
  3489. {
  3490. /* Only adjust if ZONE_MOVABLE is on this node */
  3491. if (zone_movable_pfn[nid]) {
  3492. /* Size ZONE_MOVABLE */
  3493. if (zone_type == ZONE_MOVABLE) {
  3494. *zone_start_pfn = zone_movable_pfn[nid];
  3495. *zone_end_pfn = min(node_end_pfn,
  3496. arch_zone_highest_possible_pfn[movable_zone]);
  3497. /* Adjust for ZONE_MOVABLE starting within this range */
  3498. } else if (*zone_start_pfn < zone_movable_pfn[nid] &&
  3499. *zone_end_pfn > zone_movable_pfn[nid]) {
  3500. *zone_end_pfn = zone_movable_pfn[nid];
  3501. /* Check if this whole range is within ZONE_MOVABLE */
  3502. } else if (*zone_start_pfn >= zone_movable_pfn[nid])
  3503. *zone_start_pfn = *zone_end_pfn;
  3504. }
  3505. }
  3506. /*
  3507. * Return the number of pages a zone spans in a node, including holes
  3508. * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
  3509. */
  3510. static unsigned long __meminit zone_spanned_pages_in_node(int nid,
  3511. unsigned long zone_type,
  3512. unsigned long *ignored)
  3513. {
  3514. unsigned long node_start_pfn, node_end_pfn;
  3515. unsigned long zone_start_pfn, zone_end_pfn;
  3516. /* Get the start and end of the node and zone */
  3517. get_pfn_range_for_nid(nid, &node_start_pfn, &node_end_pfn);
  3518. zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type];
  3519. zone_end_pfn = arch_zone_highest_possible_pfn[zone_type];
  3520. adjust_zone_range_for_zone_movable(nid, zone_type,
  3521. node_start_pfn, node_end_pfn,
  3522. &zone_start_pfn, &zone_end_pfn);
  3523. /* Check that this node has pages within the zone's required range */
  3524. if (zone_end_pfn < node_start_pfn || zone_start_pfn > node_end_pfn)
  3525. return 0;
  3526. /* Move the zone boundaries inside the node if necessary */
  3527. zone_end_pfn = min(zone_end_pfn, node_end_pfn);
  3528. zone_start_pfn = max(zone_start_pfn, node_start_pfn);
  3529. /* Return the spanned pages */
  3530. return zone_end_pfn - zone_start_pfn;
  3531. }
  3532. /*
  3533. * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
  3534. * then all holes in the requested range will be accounted for.
  3535. */
  3536. unsigned long __meminit __absent_pages_in_range(int nid,
  3537. unsigned long range_start_pfn,
  3538. unsigned long range_end_pfn)
  3539. {
  3540. unsigned long nr_absent = range_end_pfn - range_start_pfn;
  3541. unsigned long start_pfn, end_pfn;
  3542. int i;
  3543. for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
  3544. start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
  3545. end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
  3546. nr_absent -= end_pfn - start_pfn;
  3547. }
  3548. return nr_absent;
  3549. }
  3550. /**
  3551. * absent_pages_in_range - Return number of page frames in holes within a range
  3552. * @start_pfn: The start PFN to start searching for holes
  3553. * @end_pfn: The end PFN to stop searching for holes
  3554. *
  3555. * It returns the number of pages frames in memory holes within a range.
  3556. */
  3557. unsigned long __init absent_pages_in_range(unsigned long start_pfn,
  3558. unsigned long end_pfn)
  3559. {
  3560. return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
  3561. }
  3562. /* Return the number of page frames in holes in a zone on a node */
  3563. static unsigned long __meminit zone_absent_pages_in_node(int nid,
  3564. unsigned long zone_type,
  3565. unsigned long *ignored)
  3566. {
  3567. unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
  3568. unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
  3569. unsigned long node_start_pfn, node_end_pfn;
  3570. unsigned long zone_start_pfn, zone_end_pfn;
  3571. get_pfn_range_for_nid(nid, &node_start_pfn, &node_end_pfn);
  3572. zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
  3573. zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
  3574. adjust_zone_range_for_zone_movable(nid, zone_type,
  3575. node_start_pfn, node_end_pfn,
  3576. &zone_start_pfn, &zone_end_pfn);
  3577. return __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
  3578. }
  3579. #else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
  3580. static inline unsigned long __meminit zone_spanned_pages_in_node(int nid,
  3581. unsigned long zone_type,
  3582. unsigned long *zones_size)
  3583. {
  3584. return zones_size[zone_type];
  3585. }
  3586. static inline unsigned long __meminit zone_absent_pages_in_node(int nid,
  3587. unsigned long zone_type,
  3588. unsigned long *zholes_size)
  3589. {
  3590. if (!zholes_size)
  3591. return 0;
  3592. return zholes_size[zone_type];
  3593. }
  3594. #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
  3595. static void __meminit calculate_node_totalpages(struct pglist_data *pgdat,
  3596. unsigned long *zones_size, unsigned long *zholes_size)
  3597. {
  3598. unsigned long realtotalpages, totalpages = 0;
  3599. enum zone_type i;
  3600. for (i = 0; i < MAX_NR_ZONES; i++)
  3601. totalpages += zone_spanned_pages_in_node(pgdat->node_id, i,
  3602. zones_size);
  3603. pgdat->node_spanned_pages = totalpages;
  3604. realtotalpages = totalpages;
  3605. for (i = 0; i < MAX_NR_ZONES; i++)
  3606. realtotalpages -=
  3607. zone_absent_pages_in_node(pgdat->node_id, i,
  3608. zholes_size);
  3609. pgdat->node_present_pages = realtotalpages;
  3610. printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id,
  3611. realtotalpages);
  3612. }
  3613. #ifndef CONFIG_SPARSEMEM
  3614. /*
  3615. * Calculate the size of the zone->blockflags rounded to an unsigned long
  3616. * Start by making sure zonesize is a multiple of pageblock_order by rounding
  3617. * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
  3618. * round what is now in bits to nearest long in bits, then return it in
  3619. * bytes.
  3620. */
  3621. static unsigned long __init usemap_size(unsigned long zonesize)
  3622. {
  3623. unsigned long usemapsize;
  3624. usemapsize = roundup(zonesize, pageblock_nr_pages);
  3625. usemapsize = usemapsize >> pageblock_order;
  3626. usemapsize *= NR_PAGEBLOCK_BITS;
  3627. usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
  3628. return usemapsize / 8;
  3629. }
  3630. static void __init setup_usemap(struct pglist_data *pgdat,
  3631. struct zone *zone, unsigned long zonesize)
  3632. {
  3633. unsigned long usemapsize = usemap_size(zonesize);
  3634. zone->pageblock_flags = NULL;
  3635. if (usemapsize)
  3636. zone->pageblock_flags = alloc_bootmem_node_nopanic(pgdat,
  3637. usemapsize);
  3638. }
  3639. #else
  3640. static inline void setup_usemap(struct pglist_data *pgdat,
  3641. struct zone *zone, unsigned long zonesize) {}
  3642. #endif /* CONFIG_SPARSEMEM */
  3643. #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
  3644. /* Return a sensible default order for the pageblock size. */
  3645. static inline int pageblock_default_order(void)
  3646. {
  3647. if (HPAGE_SHIFT > PAGE_SHIFT)
  3648. return HUGETLB_PAGE_ORDER;
  3649. return MAX_ORDER-1;
  3650. }
  3651. /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
  3652. static inline void __init set_pageblock_order(unsigned int order)
  3653. {
  3654. /* Check that pageblock_nr_pages has not already been setup */
  3655. if (pageblock_order)
  3656. return;
  3657. /*
  3658. * Assume the largest contiguous order of interest is a huge page.
  3659. * This value may be variable depending on boot parameters on IA64
  3660. */
  3661. pageblock_order = order;
  3662. }
  3663. #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
  3664. /*
  3665. * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
  3666. * and pageblock_default_order() are unused as pageblock_order is set
  3667. * at compile-time. See include/linux/pageblock-flags.h for the values of
  3668. * pageblock_order based on the kernel config
  3669. */
  3670. static inline int pageblock_default_order(unsigned int order)
  3671. {
  3672. return MAX_ORDER-1;
  3673. }
  3674. #define set_pageblock_order(x) do {} while (0)
  3675. #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
  3676. /*
  3677. * Set up the zone data structures:
  3678. * - mark all pages reserved
  3679. * - mark all memory queues empty
  3680. * - clear the memory bitmaps
  3681. */
  3682. static void __paginginit free_area_init_core(struct pglist_data *pgdat,
  3683. unsigned long *zones_size, unsigned long *zholes_size)
  3684. {
  3685. enum zone_type j;
  3686. int nid = pgdat->node_id;
  3687. unsigned long zone_start_pfn = pgdat->node_start_pfn;
  3688. int ret;
  3689. pgdat_resize_init(pgdat);
  3690. pgdat->nr_zones = 0;
  3691. init_waitqueue_head(&pgdat->kswapd_wait);
  3692. pgdat->kswapd_max_order = 0;
  3693. pgdat_page_cgroup_init(pgdat);
  3694. for (j = 0; j < MAX_NR_ZONES; j++) {
  3695. struct zone *zone = pgdat->node_zones + j;
  3696. unsigned long size, realsize, memmap_pages;
  3697. enum lru_list lru;
  3698. size = zone_spanned_pages_in_node(nid, j, zones_size);
  3699. realsize = size - zone_absent_pages_in_node(nid, j,
  3700. zholes_size);
  3701. /*
  3702. * Adjust realsize so that it accounts for how much memory
  3703. * is used by this zone for memmap. This affects the watermark
  3704. * and per-cpu initialisations
  3705. */
  3706. memmap_pages =
  3707. PAGE_ALIGN(size * sizeof(struct page)) >> PAGE_SHIFT;
  3708. if (realsize >= memmap_pages) {
  3709. realsize -= memmap_pages;
  3710. if (memmap_pages)
  3711. printk(KERN_DEBUG
  3712. " %s zone: %lu pages used for memmap\n",
  3713. zone_names[j], memmap_pages);
  3714. } else
  3715. printk(KERN_WARNING
  3716. " %s zone: %lu pages exceeds realsize %lu\n",
  3717. zone_names[j], memmap_pages, realsize);
  3718. /* Account for reserved pages */
  3719. if (j == 0 && realsize > dma_reserve) {
  3720. realsize -= dma_reserve;
  3721. printk(KERN_DEBUG " %s zone: %lu pages reserved\n",
  3722. zone_names[0], dma_reserve);
  3723. }
  3724. if (!is_highmem_idx(j))
  3725. nr_kernel_pages += realsize;
  3726. nr_all_pages += realsize;
  3727. zone->spanned_pages = size;
  3728. zone->present_pages = realsize;
  3729. #ifdef CONFIG_NUMA
  3730. zone->node = nid;
  3731. zone->min_unmapped_pages = (realsize*sysctl_min_unmapped_ratio)
  3732. / 100;
  3733. zone->min_slab_pages = (realsize * sysctl_min_slab_ratio) / 100;
  3734. #endif
  3735. zone->name = zone_names[j];
  3736. spin_lock_init(&zone->lock);
  3737. spin_lock_init(&zone->lru_lock);
  3738. zone_seqlock_init(zone);
  3739. zone->zone_pgdat = pgdat;
  3740. zone_pcp_init(zone);
  3741. for_each_lru(lru)
  3742. INIT_LIST_HEAD(&zone->lruvec.lists[lru]);
  3743. zone->reclaim_stat.recent_rotated[0] = 0;
  3744. zone->reclaim_stat.recent_rotated[1] = 0;
  3745. zone->reclaim_stat.recent_scanned[0] = 0;
  3746. zone->reclaim_stat.recent_scanned[1] = 0;
  3747. zap_zone_vm_stats(zone);
  3748. zone->flags = 0;
  3749. if (!size)
  3750. continue;
  3751. set_pageblock_order(pageblock_default_order());
  3752. setup_usemap(pgdat, zone, size);
  3753. ret = init_currently_empty_zone(zone, zone_start_pfn,
  3754. size, MEMMAP_EARLY);
  3755. BUG_ON(ret);
  3756. memmap_init(size, nid, j, zone_start_pfn);
  3757. zone_start_pfn += size;
  3758. }
  3759. }
  3760. static void __init_refok alloc_node_mem_map(struct pglist_data *pgdat)
  3761. {
  3762. /* Skip empty nodes */
  3763. if (!pgdat->node_spanned_pages)
  3764. return;
  3765. #ifdef CONFIG_FLAT_NODE_MEM_MAP
  3766. /* ia64 gets its own node_mem_map, before this, without bootmem */
  3767. if (!pgdat->node_mem_map) {
  3768. unsigned long size, start, end;
  3769. struct page *map;
  3770. /*
  3771. * The zone's endpoints aren't required to be MAX_ORDER
  3772. * aligned but the node_mem_map endpoints must be in order
  3773. * for the buddy allocator to function correctly.
  3774. */
  3775. start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
  3776. end = pgdat->node_start_pfn + pgdat->node_spanned_pages;
  3777. end = ALIGN(end, MAX_ORDER_NR_PAGES);
  3778. size = (end - start) * sizeof(struct page);
  3779. map = alloc_remap(pgdat->node_id, size);
  3780. if (!map)
  3781. map = alloc_bootmem_node_nopanic(pgdat, size);
  3782. pgdat->node_mem_map = map + (pgdat->node_start_pfn - start);
  3783. }
  3784. #ifndef CONFIG_NEED_MULTIPLE_NODES
  3785. /*
  3786. * With no DISCONTIG, the global mem_map is just set as node 0's
  3787. */
  3788. if (pgdat == NODE_DATA(0)) {
  3789. mem_map = NODE_DATA(0)->node_mem_map;
  3790. #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
  3791. if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
  3792. mem_map -= (pgdat->node_start_pfn - ARCH_PFN_OFFSET);
  3793. #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
  3794. }
  3795. #endif
  3796. #endif /* CONFIG_FLAT_NODE_MEM_MAP */
  3797. }
  3798. void __paginginit free_area_init_node(int nid, unsigned long *zones_size,
  3799. unsigned long node_start_pfn, unsigned long *zholes_size)
  3800. {
  3801. pg_data_t *pgdat = NODE_DATA(nid);
  3802. pgdat->node_id = nid;
  3803. pgdat->node_start_pfn = node_start_pfn;
  3804. calculate_node_totalpages(pgdat, zones_size, zholes_size);
  3805. alloc_node_mem_map(pgdat);
  3806. #ifdef CONFIG_FLAT_NODE_MEM_MAP
  3807. printk(KERN_DEBUG "free_area_init_node: node %d, pgdat %08lx, node_mem_map %08lx\n",
  3808. nid, (unsigned long)pgdat,
  3809. (unsigned long)pgdat->node_mem_map);
  3810. #endif
  3811. free_area_init_core(pgdat, zones_size, zholes_size);
  3812. }
  3813. #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
  3814. #if MAX_NUMNODES > 1
  3815. /*
  3816. * Figure out the number of possible node ids.
  3817. */
  3818. static void __init setup_nr_node_ids(void)
  3819. {
  3820. unsigned int node;
  3821. unsigned int highest = 0;
  3822. for_each_node_mask(node, node_possible_map)
  3823. highest = node;
  3824. nr_node_ids = highest + 1;
  3825. }
  3826. #else
  3827. static inline void setup_nr_node_ids(void)
  3828. {
  3829. }
  3830. #endif
  3831. /**
  3832. * node_map_pfn_alignment - determine the maximum internode alignment
  3833. *
  3834. * This function should be called after node map is populated and sorted.
  3835. * It calculates the maximum power of two alignment which can distinguish
  3836. * all the nodes.
  3837. *
  3838. * For example, if all nodes are 1GiB and aligned to 1GiB, the return value
  3839. * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the
  3840. * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is
  3841. * shifted, 1GiB is enough and this function will indicate so.
  3842. *
  3843. * This is used to test whether pfn -> nid mapping of the chosen memory
  3844. * model has fine enough granularity to avoid incorrect mapping for the
  3845. * populated node map.
  3846. *
  3847. * Returns the determined alignment in pfn's. 0 if there is no alignment
  3848. * requirement (single node).
  3849. */
  3850. unsigned long __init node_map_pfn_alignment(void)
  3851. {
  3852. unsigned long accl_mask = 0, last_end = 0;
  3853. unsigned long start, end, mask;
  3854. int last_nid = -1;
  3855. int i, nid;
  3856. for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {
  3857. if (!start || last_nid < 0 || last_nid == nid) {
  3858. last_nid = nid;
  3859. last_end = end;
  3860. continue;
  3861. }
  3862. /*
  3863. * Start with a mask granular enough to pin-point to the
  3864. * start pfn and tick off bits one-by-one until it becomes
  3865. * too coarse to separate the current node from the last.
  3866. */
  3867. mask = ~((1 << __ffs(start)) - 1);
  3868. while (mask && last_end <= (start & (mask << 1)))
  3869. mask <<= 1;
  3870. /* accumulate all internode masks */
  3871. accl_mask |= mask;
  3872. }
  3873. /* convert mask to number of pages */
  3874. return ~accl_mask + 1;
  3875. }
  3876. /* Find the lowest pfn for a node */
  3877. static unsigned long __init find_min_pfn_for_node(int nid)
  3878. {
  3879. unsigned long min_pfn = ULONG_MAX;
  3880. unsigned long start_pfn;
  3881. int i;
  3882. for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL)
  3883. min_pfn = min(min_pfn, start_pfn);
  3884. if (min_pfn == ULONG_MAX) {
  3885. printk(KERN_WARNING
  3886. "Could not find start_pfn for node %d\n", nid);
  3887. return 0;
  3888. }
  3889. return min_pfn;
  3890. }
  3891. /**
  3892. * find_min_pfn_with_active_regions - Find the minimum PFN registered
  3893. *
  3894. * It returns the minimum PFN based on information provided via
  3895. * add_active_range().
  3896. */
  3897. unsigned long __init find_min_pfn_with_active_regions(void)
  3898. {
  3899. return find_min_pfn_for_node(MAX_NUMNODES);
  3900. }
  3901. /*
  3902. * early_calculate_totalpages()
  3903. * Sum pages in active regions for movable zone.
  3904. * Populate N_HIGH_MEMORY for calculating usable_nodes.
  3905. */
  3906. static unsigned long __init early_calculate_totalpages(void)
  3907. {
  3908. unsigned long totalpages = 0;
  3909. unsigned long start_pfn, end_pfn;
  3910. int i, nid;
  3911. for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
  3912. unsigned long pages = end_pfn - start_pfn;
  3913. totalpages += pages;
  3914. if (pages)
  3915. node_set_state(nid, N_HIGH_MEMORY);
  3916. }
  3917. return totalpages;
  3918. }
  3919. /*
  3920. * Find the PFN the Movable zone begins in each node. Kernel memory
  3921. * is spread evenly between nodes as long as the nodes have enough
  3922. * memory. When they don't, some nodes will have more kernelcore than
  3923. * others
  3924. */
  3925. static void __init find_zone_movable_pfns_for_nodes(unsigned long *movable_pfn)
  3926. {
  3927. int i, nid;
  3928. unsigned long usable_startpfn;
  3929. unsigned long kernelcore_node, kernelcore_remaining;
  3930. /* save the state before borrow the nodemask */
  3931. nodemask_t saved_node_state = node_states[N_HIGH_MEMORY];
  3932. unsigned long totalpages = early_calculate_totalpages();
  3933. int usable_nodes = nodes_weight(node_states[N_HIGH_MEMORY]);
  3934. /*
  3935. * If movablecore was specified, calculate what size of
  3936. * kernelcore that corresponds so that memory usable for
  3937. * any allocation type is evenly spread. If both kernelcore
  3938. * and movablecore are specified, then the value of kernelcore
  3939. * will be used for required_kernelcore if it's greater than
  3940. * what movablecore would have allowed.
  3941. */
  3942. if (required_movablecore) {
  3943. unsigned long corepages;
  3944. /*
  3945. * Round-up so that ZONE_MOVABLE is at least as large as what
  3946. * was requested by the user
  3947. */
  3948. required_movablecore =
  3949. roundup(required_movablecore, MAX_ORDER_NR_PAGES);
  3950. corepages = totalpages - required_movablecore;
  3951. required_kernelcore = max(required_kernelcore, corepages);
  3952. }
  3953. /* If kernelcore was not specified, there is no ZONE_MOVABLE */
  3954. if (!required_kernelcore)
  3955. goto out;
  3956. /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
  3957. find_usable_zone_for_movable();
  3958. usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
  3959. restart:
  3960. /* Spread kernelcore memory as evenly as possible throughout nodes */
  3961. kernelcore_node = required_kernelcore / usable_nodes;
  3962. for_each_node_state(nid, N_HIGH_MEMORY) {
  3963. unsigned long start_pfn, end_pfn;
  3964. /*
  3965. * Recalculate kernelcore_node if the division per node
  3966. * now exceeds what is necessary to satisfy the requested
  3967. * amount of memory for the kernel
  3968. */
  3969. if (required_kernelcore < kernelcore_node)
  3970. kernelcore_node = required_kernelcore / usable_nodes;
  3971. /*
  3972. * As the map is walked, we track how much memory is usable
  3973. * by the kernel using kernelcore_remaining. When it is
  3974. * 0, the rest of the node is usable by ZONE_MOVABLE
  3975. */
  3976. kernelcore_remaining = kernelcore_node;
  3977. /* Go through each range of PFNs within this node */
  3978. for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
  3979. unsigned long size_pages;
  3980. start_pfn = max(start_pfn, zone_movable_pfn[nid]);
  3981. if (start_pfn >= end_pfn)
  3982. continue;
  3983. /* Account for what is only usable for kernelcore */
  3984. if (start_pfn < usable_startpfn) {
  3985. unsigned long kernel_pages;
  3986. kernel_pages = min(end_pfn, usable_startpfn)
  3987. - start_pfn;
  3988. kernelcore_remaining -= min(kernel_pages,
  3989. kernelcore_remaining);
  3990. required_kernelcore -= min(kernel_pages,
  3991. required_kernelcore);
  3992. /* Continue if range is now fully accounted */
  3993. if (end_pfn <= usable_startpfn) {
  3994. /*
  3995. * Push zone_movable_pfn to the end so
  3996. * that if we have to rebalance
  3997. * kernelcore across nodes, we will
  3998. * not double account here
  3999. */
  4000. zone_movable_pfn[nid] = end_pfn;
  4001. continue;
  4002. }
  4003. start_pfn = usable_startpfn;
  4004. }
  4005. /*
  4006. * The usable PFN range for ZONE_MOVABLE is from
  4007. * start_pfn->end_pfn. Calculate size_pages as the
  4008. * number of pages used as kernelcore
  4009. */
  4010. size_pages = end_pfn - start_pfn;
  4011. if (size_pages > kernelcore_remaining)
  4012. size_pages = kernelcore_remaining;
  4013. zone_movable_pfn[nid] = start_pfn + size_pages;
  4014. /*
  4015. * Some kernelcore has been met, update counts and
  4016. * break if the kernelcore for this node has been
  4017. * satisified
  4018. */
  4019. required_kernelcore -= min(required_kernelcore,
  4020. size_pages);
  4021. kernelcore_remaining -= size_pages;
  4022. if (!kernelcore_remaining)
  4023. break;
  4024. }
  4025. }
  4026. /*
  4027. * If there is still required_kernelcore, we do another pass with one
  4028. * less node in the count. This will push zone_movable_pfn[nid] further
  4029. * along on the nodes that still have memory until kernelcore is
  4030. * satisified
  4031. */
  4032. usable_nodes--;
  4033. if (usable_nodes && required_kernelcore > usable_nodes)
  4034. goto restart;
  4035. /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
  4036. for (nid = 0; nid < MAX_NUMNODES; nid++)
  4037. zone_movable_pfn[nid] =
  4038. roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
  4039. out:
  4040. /* restore the node_state */
  4041. node_states[N_HIGH_MEMORY] = saved_node_state;
  4042. }
  4043. /* Any regular memory on that node ? */
  4044. static void check_for_regular_memory(pg_data_t *pgdat)
  4045. {
  4046. #ifdef CONFIG_HIGHMEM
  4047. enum zone_type zone_type;
  4048. for (zone_type = 0; zone_type <= ZONE_NORMAL; zone_type++) {
  4049. struct zone *zone = &pgdat->node_zones[zone_type];
  4050. if (zone->present_pages) {
  4051. node_set_state(zone_to_nid(zone), N_NORMAL_MEMORY);
  4052. break;
  4053. }
  4054. }
  4055. #endif
  4056. }
  4057. /**
  4058. * free_area_init_nodes - Initialise all pg_data_t and zone data
  4059. * @max_zone_pfn: an array of max PFNs for each zone
  4060. *
  4061. * This will call free_area_init_node() for each active node in the system.
  4062. * Using the page ranges provided by add_active_range(), the size of each
  4063. * zone in each node and their holes is calculated. If the maximum PFN
  4064. * between two adjacent zones match, it is assumed that the zone is empty.
  4065. * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
  4066. * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
  4067. * starts where the previous one ended. For example, ZONE_DMA32 starts
  4068. * at arch_max_dma_pfn.
  4069. */
  4070. void __init free_area_init_nodes(unsigned long *max_zone_pfn)
  4071. {
  4072. unsigned long start_pfn, end_pfn;
  4073. int i, nid;
  4074. /* Record where the zone boundaries are */
  4075. memset(arch_zone_lowest_possible_pfn, 0,
  4076. sizeof(arch_zone_lowest_possible_pfn));
  4077. memset(arch_zone_highest_possible_pfn, 0,
  4078. sizeof(arch_zone_highest_possible_pfn));
  4079. arch_zone_lowest_possible_pfn[0] = find_min_pfn_with_active_regions();
  4080. arch_zone_highest_possible_pfn[0] = max_zone_pfn[0];
  4081. for (i = 1; i < MAX_NR_ZONES; i++) {
  4082. if (i == ZONE_MOVABLE)
  4083. continue;
  4084. arch_zone_lowest_possible_pfn[i] =
  4085. arch_zone_highest_possible_pfn[i-1];
  4086. arch_zone_highest_possible_pfn[i] =
  4087. max(max_zone_pfn[i], arch_zone_lowest_possible_pfn[i]);
  4088. }
  4089. arch_zone_lowest_possible_pfn[ZONE_MOVABLE] = 0;
  4090. arch_zone_highest_possible_pfn[ZONE_MOVABLE] = 0;
  4091. /* Find the PFNs that ZONE_MOVABLE begins at in each node */
  4092. memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
  4093. find_zone_movable_pfns_for_nodes(zone_movable_pfn);
  4094. /* Print out the zone ranges */
  4095. printk("Zone PFN ranges:\n");
  4096. for (i = 0; i < MAX_NR_ZONES; i++) {
  4097. if (i == ZONE_MOVABLE)
  4098. continue;
  4099. printk(" %-8s ", zone_names[i]);
  4100. if (arch_zone_lowest_possible_pfn[i] ==
  4101. arch_zone_highest_possible_pfn[i])
  4102. printk("empty\n");
  4103. else
  4104. printk("%0#10lx -> %0#10lx\n",
  4105. arch_zone_lowest_possible_pfn[i],
  4106. arch_zone_highest_possible_pfn[i]);
  4107. }
  4108. /* Print out the PFNs ZONE_MOVABLE begins at in each node */
  4109. printk("Movable zone start PFN for each node\n");
  4110. for (i = 0; i < MAX_NUMNODES; i++) {
  4111. if (zone_movable_pfn[i])
  4112. printk(" Node %d: %lu\n", i, zone_movable_pfn[i]);
  4113. }
  4114. /* Print out the early_node_map[] */
  4115. printk("Early memory PFN ranges\n");
  4116. for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
  4117. printk(" %3d: %0#10lx -> %0#10lx\n", nid, start_pfn, end_pfn);
  4118. /* Initialise every node */
  4119. mminit_verify_pageflags_layout();
  4120. setup_nr_node_ids();
  4121. for_each_online_node(nid) {
  4122. pg_data_t *pgdat = NODE_DATA(nid);
  4123. free_area_init_node(nid, NULL,
  4124. find_min_pfn_for_node(nid), NULL);
  4125. /* Any memory on that node */
  4126. if (pgdat->node_present_pages)
  4127. node_set_state(nid, N_HIGH_MEMORY);
  4128. check_for_regular_memory(pgdat);
  4129. }
  4130. }
  4131. static int __init cmdline_parse_core(char *p, unsigned long *core)
  4132. {
  4133. unsigned long long coremem;
  4134. if (!p)
  4135. return -EINVAL;
  4136. coremem = memparse(p, &p);
  4137. *core = coremem >> PAGE_SHIFT;
  4138. /* Paranoid check that UL is enough for the coremem value */
  4139. WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
  4140. return 0;
  4141. }
  4142. /*
  4143. * kernelcore=size sets the amount of memory for use for allocations that
  4144. * cannot be reclaimed or migrated.
  4145. */
  4146. static int __init cmdline_parse_kernelcore(char *p)
  4147. {
  4148. return cmdline_parse_core(p, &required_kernelcore);
  4149. }
  4150. /*
  4151. * movablecore=size sets the amount of memory for use for allocations that
  4152. * can be reclaimed or migrated.
  4153. */
  4154. static int __init cmdline_parse_movablecore(char *p)
  4155. {
  4156. return cmdline_parse_core(p, &required_movablecore);
  4157. }
  4158. early_param("kernelcore", cmdline_parse_kernelcore);
  4159. early_param("movablecore", cmdline_parse_movablecore);
  4160. #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
  4161. /**
  4162. * set_dma_reserve - set the specified number of pages reserved in the first zone
  4163. * @new_dma_reserve: The number of pages to mark reserved
  4164. *
  4165. * The per-cpu batchsize and zone watermarks are determined by present_pages.
  4166. * In the DMA zone, a significant percentage may be consumed by kernel image
  4167. * and other unfreeable allocations which can skew the watermarks badly. This
  4168. * function may optionally be used to account for unfreeable pages in the
  4169. * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
  4170. * smaller per-cpu batchsize.
  4171. */
  4172. void __init set_dma_reserve(unsigned long new_dma_reserve)
  4173. {
  4174. dma_reserve = new_dma_reserve;
  4175. }
  4176. void __init free_area_init(unsigned long *zones_size)
  4177. {
  4178. free_area_init_node(0, zones_size,
  4179. __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL);
  4180. }
  4181. static int page_alloc_cpu_notify(struct notifier_block *self,
  4182. unsigned long action, void *hcpu)
  4183. {
  4184. int cpu = (unsigned long)hcpu;
  4185. if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
  4186. drain_pages(cpu);
  4187. /*
  4188. * Spill the event counters of the dead processor
  4189. * into the current processors event counters.
  4190. * This artificially elevates the count of the current
  4191. * processor.
  4192. */
  4193. vm_events_fold_cpu(cpu);
  4194. /*
  4195. * Zero the differential counters of the dead processor
  4196. * so that the vm statistics are consistent.
  4197. *
  4198. * This is only okay since the processor is dead and cannot
  4199. * race with what we are doing.
  4200. */
  4201. refresh_cpu_vm_stats(cpu);
  4202. }
  4203. return NOTIFY_OK;
  4204. }
  4205. void __init page_alloc_init(void)
  4206. {
  4207. hotcpu_notifier(page_alloc_cpu_notify, 0);
  4208. }
  4209. /*
  4210. * calculate_totalreserve_pages - called when sysctl_lower_zone_reserve_ratio
  4211. * or min_free_kbytes changes.
  4212. */
  4213. static void calculate_totalreserve_pages(void)
  4214. {
  4215. struct pglist_data *pgdat;
  4216. unsigned long reserve_pages = 0;
  4217. enum zone_type i, j;
  4218. for_each_online_pgdat(pgdat) {
  4219. for (i = 0; i < MAX_NR_ZONES; i++) {
  4220. struct zone *zone = pgdat->node_zones + i;
  4221. unsigned long max = 0;
  4222. /* Find valid and maximum lowmem_reserve in the zone */
  4223. for (j = i; j < MAX_NR_ZONES; j++) {
  4224. if (zone->lowmem_reserve[j] > max)
  4225. max = zone->lowmem_reserve[j];
  4226. }
  4227. /* we treat the high watermark as reserved pages. */
  4228. max += high_wmark_pages(zone);
  4229. if (max > zone->present_pages)
  4230. max = zone->present_pages;
  4231. reserve_pages += max;
  4232. /*
  4233. * Lowmem reserves are not available to
  4234. * GFP_HIGHUSER page cache allocations and
  4235. * kswapd tries to balance zones to their high
  4236. * watermark. As a result, neither should be
  4237. * regarded as dirtyable memory, to prevent a
  4238. * situation where reclaim has to clean pages
  4239. * in order to balance the zones.
  4240. */
  4241. zone->dirty_balance_reserve = max;
  4242. }
  4243. }
  4244. dirty_balance_reserve = reserve_pages;
  4245. totalreserve_pages = reserve_pages;
  4246. }
  4247. /*
  4248. * setup_per_zone_lowmem_reserve - called whenever
  4249. * sysctl_lower_zone_reserve_ratio changes. Ensures that each zone
  4250. * has a correct pages reserved value, so an adequate number of
  4251. * pages are left in the zone after a successful __alloc_pages().
  4252. */
  4253. static void setup_per_zone_lowmem_reserve(void)
  4254. {
  4255. struct pglist_data *pgdat;
  4256. enum zone_type j, idx;
  4257. for_each_online_pgdat(pgdat) {
  4258. for (j = 0; j < MAX_NR_ZONES; j++) {
  4259. struct zone *zone = pgdat->node_zones + j;
  4260. unsigned long present_pages = zone->present_pages;
  4261. zone->lowmem_reserve[j] = 0;
  4262. idx = j;
  4263. while (idx) {
  4264. struct zone *lower_zone;
  4265. idx--;
  4266. if (sysctl_lowmem_reserve_ratio[idx] < 1)
  4267. sysctl_lowmem_reserve_ratio[idx] = 1;
  4268. lower_zone = pgdat->node_zones + idx;
  4269. lower_zone->lowmem_reserve[j] = present_pages /
  4270. sysctl_lowmem_reserve_ratio[idx];
  4271. present_pages += lower_zone->present_pages;
  4272. }
  4273. }
  4274. }
  4275. /* update totalreserve_pages */
  4276. calculate_totalreserve_pages();
  4277. }
  4278. /**
  4279. * setup_per_zone_wmarks - called when min_free_kbytes changes
  4280. * or when memory is hot-{added|removed}
  4281. *
  4282. * Ensures that the watermark[min,low,high] values for each zone are set
  4283. * correctly with respect to min_free_kbytes.
  4284. */
  4285. void setup_per_zone_wmarks(void)
  4286. {
  4287. unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
  4288. unsigned long lowmem_pages = 0;
  4289. struct zone *zone;
  4290. unsigned long flags;
  4291. /* Calculate total number of !ZONE_HIGHMEM pages */
  4292. for_each_zone(zone) {
  4293. if (!is_highmem(zone))
  4294. lowmem_pages += zone->present_pages;
  4295. }
  4296. for_each_zone(zone) {
  4297. u64 tmp;
  4298. spin_lock_irqsave(&zone->lock, flags);
  4299. tmp = (u64)pages_min * zone->present_pages;
  4300. do_div(tmp, lowmem_pages);
  4301. if (is_highmem(zone)) {
  4302. /*
  4303. * __GFP_HIGH and PF_MEMALLOC allocations usually don't
  4304. * need highmem pages, so cap pages_min to a small
  4305. * value here.
  4306. *
  4307. * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
  4308. * deltas controls asynch page reclaim, and so should
  4309. * not be capped for highmem.
  4310. */
  4311. int min_pages;
  4312. min_pages = zone->present_pages / 1024;
  4313. if (min_pages < SWAP_CLUSTER_MAX)
  4314. min_pages = SWAP_CLUSTER_MAX;
  4315. if (min_pages > 128)
  4316. min_pages = 128;
  4317. zone->watermark[WMARK_MIN] = min_pages;
  4318. } else {
  4319. /*
  4320. * If it's a lowmem zone, reserve a number of pages
  4321. * proportionate to the zone's size.
  4322. */
  4323. zone->watermark[WMARK_MIN] = tmp;
  4324. }
  4325. zone->watermark[WMARK_LOW] = min_wmark_pages(zone) + (tmp >> 2);
  4326. zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + (tmp >> 1);
  4327. setup_zone_migrate_reserve(zone);
  4328. spin_unlock_irqrestore(&zone->lock, flags);
  4329. }
  4330. /* update totalreserve_pages */
  4331. calculate_totalreserve_pages();
  4332. }
  4333. /*
  4334. * The inactive anon list should be small enough that the VM never has to
  4335. * do too much work, but large enough that each inactive page has a chance
  4336. * to be referenced again before it is swapped out.
  4337. *
  4338. * The inactive_anon ratio is the target ratio of ACTIVE_ANON to
  4339. * INACTIVE_ANON pages on this zone's LRU, maintained by the
  4340. * pageout code. A zone->inactive_ratio of 3 means 3:1 or 25% of
  4341. * the anonymous pages are kept on the inactive list.
  4342. *
  4343. * total target max
  4344. * memory ratio inactive anon
  4345. * -------------------------------------
  4346. * 10MB 1 5MB
  4347. * 100MB 1 50MB
  4348. * 1GB 3 250MB
  4349. * 10GB 10 0.9GB
  4350. * 100GB 31 3GB
  4351. * 1TB 101 10GB
  4352. * 10TB 320 32GB
  4353. */
  4354. static void __meminit calculate_zone_inactive_ratio(struct zone *zone)
  4355. {
  4356. unsigned int gb, ratio;
  4357. /* Zone size in gigabytes */
  4358. gb = zone->present_pages >> (30 - PAGE_SHIFT);
  4359. if (gb)
  4360. ratio = int_sqrt(10 * gb);
  4361. else
  4362. ratio = 1;
  4363. zone->inactive_ratio = ratio;
  4364. }
  4365. static void __meminit setup_per_zone_inactive_ratio(void)
  4366. {
  4367. struct zone *zone;
  4368. for_each_zone(zone)
  4369. calculate_zone_inactive_ratio(zone);
  4370. }
  4371. /*
  4372. * Initialise min_free_kbytes.
  4373. *
  4374. * For small machines we want it small (128k min). For large machines
  4375. * we want it large (64MB max). But it is not linear, because network
  4376. * bandwidth does not increase linearly with machine size. We use
  4377. *
  4378. * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy:
  4379. * min_free_kbytes = sqrt(lowmem_kbytes * 16)
  4380. *
  4381. * which yields
  4382. *
  4383. * 16MB: 512k
  4384. * 32MB: 724k
  4385. * 64MB: 1024k
  4386. * 128MB: 1448k
  4387. * 256MB: 2048k
  4388. * 512MB: 2896k
  4389. * 1024MB: 4096k
  4390. * 2048MB: 5792k
  4391. * 4096MB: 8192k
  4392. * 8192MB: 11584k
  4393. * 16384MB: 16384k
  4394. */
  4395. int __meminit init_per_zone_wmark_min(void)
  4396. {
  4397. unsigned long lowmem_kbytes;
  4398. lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
  4399. min_free_kbytes = int_sqrt(lowmem_kbytes * 16);
  4400. if (min_free_kbytes < 128)
  4401. min_free_kbytes = 128;
  4402. if (min_free_kbytes > 65536)
  4403. min_free_kbytes = 65536;
  4404. setup_per_zone_wmarks();
  4405. refresh_zone_stat_thresholds();
  4406. setup_per_zone_lowmem_reserve();
  4407. setup_per_zone_inactive_ratio();
  4408. return 0;
  4409. }
  4410. module_init(init_per_zone_wmark_min)
  4411. /*
  4412. * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so
  4413. * that we can call two helper functions whenever min_free_kbytes
  4414. * changes.
  4415. */
  4416. int min_free_kbytes_sysctl_handler(ctl_table *table, int write,
  4417. void __user *buffer, size_t *length, loff_t *ppos)
  4418. {
  4419. proc_dointvec(table, write, buffer, length, ppos);
  4420. if (write)
  4421. setup_per_zone_wmarks();
  4422. return 0;
  4423. }
  4424. #ifdef CONFIG_NUMA
  4425. int sysctl_min_unmapped_ratio_sysctl_handler(ctl_table *table, int write,
  4426. void __user *buffer, size_t *length, loff_t *ppos)
  4427. {
  4428. struct zone *zone;
  4429. int rc;
  4430. rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
  4431. if (rc)
  4432. return rc;
  4433. for_each_zone(zone)
  4434. zone->min_unmapped_pages = (zone->present_pages *
  4435. sysctl_min_unmapped_ratio) / 100;
  4436. return 0;
  4437. }
  4438. int sysctl_min_slab_ratio_sysctl_handler(ctl_table *table, int write,
  4439. void __user *buffer, size_t *length, loff_t *ppos)
  4440. {
  4441. struct zone *zone;
  4442. int rc;
  4443. rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
  4444. if (rc)
  4445. return rc;
  4446. for_each_zone(zone)
  4447. zone->min_slab_pages = (zone->present_pages *
  4448. sysctl_min_slab_ratio) / 100;
  4449. return 0;
  4450. }
  4451. #endif
  4452. /*
  4453. * lowmem_reserve_ratio_sysctl_handler - just a wrapper around
  4454. * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve()
  4455. * whenever sysctl_lowmem_reserve_ratio changes.
  4456. *
  4457. * The reserve ratio obviously has absolutely no relation with the
  4458. * minimum watermarks. The lowmem reserve ratio can only make sense
  4459. * if in function of the boot time zone sizes.
  4460. */
  4461. int lowmem_reserve_ratio_sysctl_handler(ctl_table *table, int write,
  4462. void __user *buffer, size_t *length, loff_t *ppos)
  4463. {
  4464. proc_dointvec_minmax(table, write, buffer, length, ppos);
  4465. setup_per_zone_lowmem_reserve();
  4466. return 0;
  4467. }
  4468. /*
  4469. * percpu_pagelist_fraction - changes the pcp->high for each zone on each
  4470. * cpu. It is the fraction of total pages in each zone that a hot per cpu pagelist
  4471. * can have before it gets flushed back to buddy allocator.
  4472. */
  4473. int percpu_pagelist_fraction_sysctl_handler(ctl_table *table, int write,
  4474. void __user *buffer, size_t *length, loff_t *ppos)
  4475. {
  4476. struct zone *zone;
  4477. unsigned int cpu;
  4478. int ret;
  4479. ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
  4480. if (!write || (ret == -EINVAL))
  4481. return ret;
  4482. for_each_populated_zone(zone) {
  4483. for_each_possible_cpu(cpu) {
  4484. unsigned long high;
  4485. high = zone->present_pages / percpu_pagelist_fraction;
  4486. setup_pagelist_highmark(
  4487. per_cpu_ptr(zone->pageset, cpu), high);
  4488. }
  4489. }
  4490. return 0;
  4491. }
  4492. int hashdist = HASHDIST_DEFAULT;
  4493. #ifdef CONFIG_NUMA
  4494. static int __init set_hashdist(char *str)
  4495. {
  4496. if (!str)
  4497. return 0;
  4498. hashdist = simple_strtoul(str, &str, 0);
  4499. return 1;
  4500. }
  4501. __setup("hashdist=", set_hashdist);
  4502. #endif
  4503. /*
  4504. * allocate a large system hash table from bootmem
  4505. * - it is assumed that the hash table must contain an exact power-of-2
  4506. * quantity of entries
  4507. * - limit is the number of hash buckets, not the total allocation size
  4508. */
  4509. void *__init alloc_large_system_hash(const char *tablename,
  4510. unsigned long bucketsize,
  4511. unsigned long numentries,
  4512. int scale,
  4513. int flags,
  4514. unsigned int *_hash_shift,
  4515. unsigned int *_hash_mask,
  4516. unsigned long limit)
  4517. {
  4518. unsigned long long max = limit;
  4519. unsigned long log2qty, size;
  4520. void *table = NULL;
  4521. /* allow the kernel cmdline to have a say */
  4522. if (!numentries) {
  4523. /* round applicable memory size up to nearest megabyte */
  4524. numentries = nr_kernel_pages;
  4525. numentries += (1UL << (20 - PAGE_SHIFT)) - 1;
  4526. numentries >>= 20 - PAGE_SHIFT;
  4527. numentries <<= 20 - PAGE_SHIFT;
  4528. /* limit to 1 bucket per 2^scale bytes of low memory */
  4529. if (scale > PAGE_SHIFT)
  4530. numentries >>= (scale - PAGE_SHIFT);
  4531. else
  4532. numentries <<= (PAGE_SHIFT - scale);
  4533. /* Make sure we've got at least a 0-order allocation.. */
  4534. if (unlikely(flags & HASH_SMALL)) {
  4535. /* Makes no sense without HASH_EARLY */
  4536. WARN_ON(!(flags & HASH_EARLY));
  4537. if (!(numentries >> *_hash_shift)) {
  4538. numentries = 1UL << *_hash_shift;
  4539. BUG_ON(!numentries);
  4540. }
  4541. } else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
  4542. numentries = PAGE_SIZE / bucketsize;
  4543. }
  4544. numentries = roundup_pow_of_two(numentries);
  4545. /* limit allocation size to 1/16 total memory by default */
  4546. if (max == 0) {
  4547. max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
  4548. do_div(max, bucketsize);
  4549. }
  4550. max = min(max, 0x80000000ULL);
  4551. if (numentries > max)
  4552. numentries = max;
  4553. log2qty = ilog2(numentries);
  4554. do {
  4555. size = bucketsize << log2qty;
  4556. if (flags & HASH_EARLY)
  4557. table = alloc_bootmem_nopanic(size);
  4558. else if (hashdist)
  4559. table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL);
  4560. else {
  4561. /*
  4562. * If bucketsize is not a power-of-two, we may free
  4563. * some pages at the end of hash table which
  4564. * alloc_pages_exact() automatically does
  4565. */
  4566. if (get_order(size) < MAX_ORDER) {
  4567. table = alloc_pages_exact(size, GFP_ATOMIC);
  4568. kmemleak_alloc(table, size, 1, GFP_ATOMIC);
  4569. }
  4570. }
  4571. } while (!table && size > PAGE_SIZE && --log2qty);
  4572. if (!table)
  4573. panic("Failed to allocate %s hash table\n", tablename);
  4574. printk(KERN_INFO "%s hash table entries: %ld (order: %d, %lu bytes)\n",
  4575. tablename,
  4576. (1UL << log2qty),
  4577. ilog2(size) - PAGE_SHIFT,
  4578. size);
  4579. if (_hash_shift)
  4580. *_hash_shift = log2qty;
  4581. if (_hash_mask)
  4582. *_hash_mask = (1 << log2qty) - 1;
  4583. return table;
  4584. }
  4585. /* Return a pointer to the bitmap storing bits affecting a block of pages */
  4586. static inline unsigned long *get_pageblock_bitmap(struct zone *zone,
  4587. unsigned long pfn)
  4588. {
  4589. #ifdef CONFIG_SPARSEMEM
  4590. return __pfn_to_section(pfn)->pageblock_flags;
  4591. #else
  4592. return zone->pageblock_flags;
  4593. #endif /* CONFIG_SPARSEMEM */
  4594. }
  4595. static inline int pfn_to_bitidx(struct zone *zone, unsigned long pfn)
  4596. {
  4597. #ifdef CONFIG_SPARSEMEM
  4598. pfn &= (PAGES_PER_SECTION-1);
  4599. return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
  4600. #else
  4601. pfn = pfn - zone->zone_start_pfn;
  4602. return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
  4603. #endif /* CONFIG_SPARSEMEM */
  4604. }
  4605. /**
  4606. * get_pageblock_flags_group - Return the requested group of flags for the pageblock_nr_pages block of pages
  4607. * @page: The page within the block of interest
  4608. * @start_bitidx: The first bit of interest to retrieve
  4609. * @end_bitidx: The last bit of interest
  4610. * returns pageblock_bits flags
  4611. */
  4612. unsigned long get_pageblock_flags_group(struct page *page,
  4613. int start_bitidx, int end_bitidx)
  4614. {
  4615. struct zone *zone;
  4616. unsigned long *bitmap;
  4617. unsigned long pfn, bitidx;
  4618. unsigned long flags = 0;
  4619. unsigned long value = 1;
  4620. zone = page_zone(page);
  4621. pfn = page_to_pfn(page);
  4622. bitmap = get_pageblock_bitmap(zone, pfn);
  4623. bitidx = pfn_to_bitidx(zone, pfn);
  4624. for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1)
  4625. if (test_bit(bitidx + start_bitidx, bitmap))
  4626. flags |= value;
  4627. return flags;
  4628. }
  4629. /**
  4630. * set_pageblock_flags_group - Set the requested group of flags for a pageblock_nr_pages block of pages
  4631. * @page: The page within the block of interest
  4632. * @start_bitidx: The first bit of interest
  4633. * @end_bitidx: The last bit of interest
  4634. * @flags: The flags to set
  4635. */
  4636. void set_pageblock_flags_group(struct page *page, unsigned long flags,
  4637. int start_bitidx, int end_bitidx)
  4638. {
  4639. struct zone *zone;
  4640. unsigned long *bitmap;
  4641. unsigned long pfn, bitidx;
  4642. unsigned long value = 1;
  4643. zone = page_zone(page);
  4644. pfn = page_to_pfn(page);
  4645. bitmap = get_pageblock_bitmap(zone, pfn);
  4646. bitidx = pfn_to_bitidx(zone, pfn);
  4647. VM_BUG_ON(pfn < zone->zone_start_pfn);
  4648. VM_BUG_ON(pfn >= zone->zone_start_pfn + zone->spanned_pages);
  4649. for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1)
  4650. if (flags & value)
  4651. __set_bit(bitidx + start_bitidx, bitmap);
  4652. else
  4653. __clear_bit(bitidx + start_bitidx, bitmap);
  4654. }
  4655. /*
  4656. * This is designed as sub function...plz see page_isolation.c also.
  4657. * set/clear page block's type to be ISOLATE.
  4658. * page allocater never alloc memory from ISOLATE block.
  4659. */
  4660. static int
  4661. __count_immobile_pages(struct zone *zone, struct page *page, int count)
  4662. {
  4663. unsigned long pfn, iter, found;
  4664. /*
  4665. * For avoiding noise data, lru_add_drain_all() should be called
  4666. * If ZONE_MOVABLE, the zone never contains immobile pages
  4667. */
  4668. if (zone_idx(zone) == ZONE_MOVABLE)
  4669. return true;
  4670. if (get_pageblock_migratetype(page) == MIGRATE_MOVABLE)
  4671. return true;
  4672. pfn = page_to_pfn(page);
  4673. for (found = 0, iter = 0; iter < pageblock_nr_pages; iter++) {
  4674. unsigned long check = pfn + iter;
  4675. if (!pfn_valid_within(check))
  4676. continue;
  4677. page = pfn_to_page(check);
  4678. if (!page_count(page)) {
  4679. if (PageBuddy(page))
  4680. iter += (1 << page_order(page)) - 1;
  4681. continue;
  4682. }
  4683. if (!PageLRU(page))
  4684. found++;
  4685. /*
  4686. * If there are RECLAIMABLE pages, we need to check it.
  4687. * But now, memory offline itself doesn't call shrink_slab()
  4688. * and it still to be fixed.
  4689. */
  4690. /*
  4691. * If the page is not RAM, page_count()should be 0.
  4692. * we don't need more check. This is an _used_ not-movable page.
  4693. *
  4694. * The problematic thing here is PG_reserved pages. PG_reserved
  4695. * is set to both of a memory hole page and a _used_ kernel
  4696. * page at boot.
  4697. */
  4698. if (found > count)
  4699. return false;
  4700. }
  4701. return true;
  4702. }
  4703. bool is_pageblock_removable_nolock(struct page *page)
  4704. {
  4705. struct zone *zone;
  4706. unsigned long pfn;
  4707. /*
  4708. * We have to be careful here because we are iterating over memory
  4709. * sections which are not zone aware so we might end up outside of
  4710. * the zone but still within the section.
  4711. * We have to take care about the node as well. If the node is offline
  4712. * its NODE_DATA will be NULL - see page_zone.
  4713. */
  4714. if (!node_online(page_to_nid(page)))
  4715. return false;
  4716. zone = page_zone(page);
  4717. pfn = page_to_pfn(page);
  4718. if (zone->zone_start_pfn > pfn ||
  4719. zone->zone_start_pfn + zone->spanned_pages <= pfn)
  4720. return false;
  4721. return __count_immobile_pages(zone, page, 0);
  4722. }
  4723. int set_migratetype_isolate(struct page *page)
  4724. {
  4725. struct zone *zone;
  4726. unsigned long flags, pfn;
  4727. struct memory_isolate_notify arg;
  4728. int notifier_ret;
  4729. int ret = -EBUSY;
  4730. zone = page_zone(page);
  4731. spin_lock_irqsave(&zone->lock, flags);
  4732. pfn = page_to_pfn(page);
  4733. arg.start_pfn = pfn;
  4734. arg.nr_pages = pageblock_nr_pages;
  4735. arg.pages_found = 0;
  4736. /*
  4737. * It may be possible to isolate a pageblock even if the
  4738. * migratetype is not MIGRATE_MOVABLE. The memory isolation
  4739. * notifier chain is used by balloon drivers to return the
  4740. * number of pages in a range that are held by the balloon
  4741. * driver to shrink memory. If all the pages are accounted for
  4742. * by balloons, are free, or on the LRU, isolation can continue.
  4743. * Later, for example, when memory hotplug notifier runs, these
  4744. * pages reported as "can be isolated" should be isolated(freed)
  4745. * by the balloon driver through the memory notifier chain.
  4746. */
  4747. notifier_ret = memory_isolate_notify(MEM_ISOLATE_COUNT, &arg);
  4748. notifier_ret = notifier_to_errno(notifier_ret);
  4749. if (notifier_ret)
  4750. goto out;
  4751. /*
  4752. * FIXME: Now, memory hotplug doesn't call shrink_slab() by itself.
  4753. * We just check MOVABLE pages.
  4754. */
  4755. if (__count_immobile_pages(zone, page, arg.pages_found))
  4756. ret = 0;
  4757. /*
  4758. * immobile means "not-on-lru" paes. If immobile is larger than
  4759. * removable-by-driver pages reported by notifier, we'll fail.
  4760. */
  4761. out:
  4762. if (!ret) {
  4763. set_pageblock_migratetype(page, MIGRATE_ISOLATE);
  4764. move_freepages_block(zone, page, MIGRATE_ISOLATE);
  4765. }
  4766. spin_unlock_irqrestore(&zone->lock, flags);
  4767. if (!ret)
  4768. drain_all_pages();
  4769. return ret;
  4770. }
  4771. void unset_migratetype_isolate(struct page *page)
  4772. {
  4773. struct zone *zone;
  4774. unsigned long flags;
  4775. zone = page_zone(page);
  4776. spin_lock_irqsave(&zone->lock, flags);
  4777. if (get_pageblock_migratetype(page) != MIGRATE_ISOLATE)
  4778. goto out;
  4779. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  4780. move_freepages_block(zone, page, MIGRATE_MOVABLE);
  4781. out:
  4782. spin_unlock_irqrestore(&zone->lock, flags);
  4783. }
  4784. #ifdef CONFIG_MEMORY_HOTREMOVE
  4785. /*
  4786. * All pages in the range must be isolated before calling this.
  4787. */
  4788. void
  4789. __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
  4790. {
  4791. struct page *page;
  4792. struct zone *zone;
  4793. int order, i;
  4794. unsigned long pfn;
  4795. unsigned long flags;
  4796. /* find the first valid pfn */
  4797. for (pfn = start_pfn; pfn < end_pfn; pfn++)
  4798. if (pfn_valid(pfn))
  4799. break;
  4800. if (pfn == end_pfn)
  4801. return;
  4802. zone = page_zone(pfn_to_page(pfn));
  4803. spin_lock_irqsave(&zone->lock, flags);
  4804. pfn = start_pfn;
  4805. while (pfn < end_pfn) {
  4806. if (!pfn_valid(pfn)) {
  4807. pfn++;
  4808. continue;
  4809. }
  4810. page = pfn_to_page(pfn);
  4811. BUG_ON(page_count(page));
  4812. BUG_ON(!PageBuddy(page));
  4813. order = page_order(page);
  4814. #ifdef CONFIG_DEBUG_VM
  4815. printk(KERN_INFO "remove from free list %lx %d %lx\n",
  4816. pfn, 1 << order, end_pfn);
  4817. #endif
  4818. list_del(&page->lru);
  4819. rmv_page_order(page);
  4820. zone->free_area[order].nr_free--;
  4821. __mod_zone_page_state(zone, NR_FREE_PAGES,
  4822. - (1UL << order));
  4823. for (i = 0; i < (1 << order); i++)
  4824. SetPageReserved((page+i));
  4825. pfn += (1 << order);
  4826. }
  4827. spin_unlock_irqrestore(&zone->lock, flags);
  4828. }
  4829. #endif
  4830. #ifdef CONFIG_MEMORY_FAILURE
  4831. bool is_free_buddy_page(struct page *page)
  4832. {
  4833. struct zone *zone = page_zone(page);
  4834. unsigned long pfn = page_to_pfn(page);
  4835. unsigned long flags;
  4836. int order;
  4837. spin_lock_irqsave(&zone->lock, flags);
  4838. for (order = 0; order < MAX_ORDER; order++) {
  4839. struct page *page_head = page - (pfn & ((1 << order) - 1));
  4840. if (PageBuddy(page_head) && page_order(page_head) >= order)
  4841. break;
  4842. }
  4843. spin_unlock_irqrestore(&zone->lock, flags);
  4844. return order < MAX_ORDER;
  4845. }
  4846. #endif
  4847. static struct trace_print_flags pageflag_names[] = {
  4848. {1UL << PG_locked, "locked" },
  4849. {1UL << PG_error, "error" },
  4850. {1UL << PG_referenced, "referenced" },
  4851. {1UL << PG_uptodate, "uptodate" },
  4852. {1UL << PG_dirty, "dirty" },
  4853. {1UL << PG_lru, "lru" },
  4854. {1UL << PG_active, "active" },
  4855. {1UL << PG_slab, "slab" },
  4856. {1UL << PG_owner_priv_1, "owner_priv_1" },
  4857. {1UL << PG_arch_1, "arch_1" },
  4858. {1UL << PG_reserved, "reserved" },
  4859. {1UL << PG_private, "private" },
  4860. {1UL << PG_private_2, "private_2" },
  4861. {1UL << PG_writeback, "writeback" },
  4862. #ifdef CONFIG_PAGEFLAGS_EXTENDED
  4863. {1UL << PG_head, "head" },
  4864. {1UL << PG_tail, "tail" },
  4865. #else
  4866. {1UL << PG_compound, "compound" },
  4867. #endif
  4868. {1UL << PG_swapcache, "swapcache" },
  4869. {1UL << PG_mappedtodisk, "mappedtodisk" },
  4870. {1UL << PG_reclaim, "reclaim" },
  4871. {1UL << PG_swapbacked, "swapbacked" },
  4872. {1UL << PG_unevictable, "unevictable" },
  4873. #ifdef CONFIG_MMU
  4874. {1UL << PG_mlocked, "mlocked" },
  4875. #endif
  4876. #ifdef CONFIG_ARCH_USES_PG_UNCACHED
  4877. {1UL << PG_uncached, "uncached" },
  4878. #endif
  4879. #ifdef CONFIG_MEMORY_FAILURE
  4880. {1UL << PG_hwpoison, "hwpoison" },
  4881. #endif
  4882. {-1UL, NULL },
  4883. };
  4884. static void dump_page_flags(unsigned long flags)
  4885. {
  4886. const char *delim = "";
  4887. unsigned long mask;
  4888. int i;
  4889. printk(KERN_ALERT "page flags: %#lx(", flags);
  4890. /* remove zone id */
  4891. flags &= (1UL << NR_PAGEFLAGS) - 1;
  4892. for (i = 0; pageflag_names[i].name && flags; i++) {
  4893. mask = pageflag_names[i].mask;
  4894. if ((flags & mask) != mask)
  4895. continue;
  4896. flags &= ~mask;
  4897. printk("%s%s", delim, pageflag_names[i].name);
  4898. delim = "|";
  4899. }
  4900. /* check for left over flags */
  4901. if (flags)
  4902. printk("%s%#lx", delim, flags);
  4903. printk(")\n");
  4904. }
  4905. void dump_page(struct page *page)
  4906. {
  4907. printk(KERN_ALERT
  4908. "page:%p count:%d mapcount:%d mapping:%p index:%#lx\n",
  4909. page, atomic_read(&page->_count), page_mapcount(page),
  4910. page->mapping, page->index);
  4911. dump_page_flags(page->flags);
  4912. mem_cgroup_print_bad_page(page);
  4913. }