page_alloc.c 176 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383
  1. /*
  2. * linux/mm/page_alloc.c
  3. *
  4. * Manages the free list, the system allocates free pages here.
  5. * Note that kmalloc() lives in slab.c
  6. *
  7. * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
  8. * Swap reorganised 29.12.95, Stephen Tweedie
  9. * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
  10. * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
  11. * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
  12. * Zone balancing, Kanoj Sarcar, SGI, Jan 2000
  13. * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
  14. * (lots of bits borrowed from Ingo Molnar & Andrew Morton)
  15. */
  16. #include <linux/stddef.h>
  17. #include <linux/mm.h>
  18. #include <linux/swap.h>
  19. #include <linux/interrupt.h>
  20. #include <linux/pagemap.h>
  21. #include <linux/jiffies.h>
  22. #include <linux/bootmem.h>
  23. #include <linux/memblock.h>
  24. #include <linux/compiler.h>
  25. #include <linux/kernel.h>
  26. #include <linux/kmemcheck.h>
  27. #include <linux/module.h>
  28. #include <linux/suspend.h>
  29. #include <linux/pagevec.h>
  30. #include <linux/blkdev.h>
  31. #include <linux/slab.h>
  32. #include <linux/ratelimit.h>
  33. #include <linux/oom.h>
  34. #include <linux/notifier.h>
  35. #include <linux/topology.h>
  36. #include <linux/sysctl.h>
  37. #include <linux/cpu.h>
  38. #include <linux/cpuset.h>
  39. #include <linux/memory_hotplug.h>
  40. #include <linux/nodemask.h>
  41. #include <linux/vmalloc.h>
  42. #include <linux/vmstat.h>
  43. #include <linux/mempolicy.h>
  44. #include <linux/stop_machine.h>
  45. #include <linux/sort.h>
  46. #include <linux/pfn.h>
  47. #include <linux/backing-dev.h>
  48. #include <linux/fault-inject.h>
  49. #include <linux/page-isolation.h>
  50. #include <linux/page_cgroup.h>
  51. #include <linux/debugobjects.h>
  52. #include <linux/kmemleak.h>
  53. #include <linux/compaction.h>
  54. #include <trace/events/kmem.h>
  55. #include <linux/ftrace_event.h>
  56. #include <linux/memcontrol.h>
  57. #include <linux/prefetch.h>
  58. #include <linux/migrate.h>
  59. #include <linux/page-debug-flags.h>
  60. #include <linux/sched/rt.h>
  61. #include <asm/tlbflush.h>
  62. #include <asm/div64.h>
  63. #include "internal.h"
  64. #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
  65. DEFINE_PER_CPU(int, numa_node);
  66. EXPORT_PER_CPU_SYMBOL(numa_node);
  67. #endif
  68. #ifdef CONFIG_HAVE_MEMORYLESS_NODES
  69. /*
  70. * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
  71. * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
  72. * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
  73. * defined in <linux/topology.h>.
  74. */
  75. DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */
  76. EXPORT_PER_CPU_SYMBOL(_numa_mem_);
  77. #endif
  78. /*
  79. * Array of node states.
  80. */
  81. nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
  82. [N_POSSIBLE] = NODE_MASK_ALL,
  83. [N_ONLINE] = { { [0] = 1UL } },
  84. #ifndef CONFIG_NUMA
  85. [N_NORMAL_MEMORY] = { { [0] = 1UL } },
  86. #ifdef CONFIG_HIGHMEM
  87. [N_HIGH_MEMORY] = { { [0] = 1UL } },
  88. #endif
  89. #ifdef CONFIG_MOVABLE_NODE
  90. [N_MEMORY] = { { [0] = 1UL } },
  91. #endif
  92. [N_CPU] = { { [0] = 1UL } },
  93. #endif /* NUMA */
  94. };
  95. EXPORT_SYMBOL(node_states);
  96. unsigned long totalram_pages __read_mostly;
  97. unsigned long totalreserve_pages __read_mostly;
  98. /*
  99. * When calculating the number of globally allowed dirty pages, there
  100. * is a certain number of per-zone reserves that should not be
  101. * considered dirtyable memory. This is the sum of those reserves
  102. * over all existing zones that contribute dirtyable memory.
  103. */
  104. unsigned long dirty_balance_reserve __read_mostly;
  105. int percpu_pagelist_fraction;
  106. gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
  107. #ifdef CONFIG_PM_SLEEP
  108. /*
  109. * The following functions are used by the suspend/hibernate code to temporarily
  110. * change gfp_allowed_mask in order to avoid using I/O during memory allocations
  111. * while devices are suspended. To avoid races with the suspend/hibernate code,
  112. * they should always be called with pm_mutex held (gfp_allowed_mask also should
  113. * only be modified with pm_mutex held, unless the suspend/hibernate code is
  114. * guaranteed not to run in parallel with that modification).
  115. */
  116. static gfp_t saved_gfp_mask;
  117. void pm_restore_gfp_mask(void)
  118. {
  119. WARN_ON(!mutex_is_locked(&pm_mutex));
  120. if (saved_gfp_mask) {
  121. gfp_allowed_mask = saved_gfp_mask;
  122. saved_gfp_mask = 0;
  123. }
  124. }
  125. void pm_restrict_gfp_mask(void)
  126. {
  127. WARN_ON(!mutex_is_locked(&pm_mutex));
  128. WARN_ON(saved_gfp_mask);
  129. saved_gfp_mask = gfp_allowed_mask;
  130. gfp_allowed_mask &= ~GFP_IOFS;
  131. }
  132. bool pm_suspended_storage(void)
  133. {
  134. if ((gfp_allowed_mask & GFP_IOFS) == GFP_IOFS)
  135. return false;
  136. return true;
  137. }
  138. #endif /* CONFIG_PM_SLEEP */
  139. #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
  140. int pageblock_order __read_mostly;
  141. #endif
  142. static void __free_pages_ok(struct page *page, unsigned int order);
  143. /*
  144. * results with 256, 32 in the lowmem_reserve sysctl:
  145. * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
  146. * 1G machine -> (16M dma, 784M normal, 224M high)
  147. * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
  148. * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
  149. * HIGHMEM allocation will (224M+784M)/256 of ram reserved in ZONE_DMA
  150. *
  151. * TBD: should special case ZONE_DMA32 machines here - in those we normally
  152. * don't need any ZONE_NORMAL reservation
  153. */
  154. int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = {
  155. #ifdef CONFIG_ZONE_DMA
  156. 256,
  157. #endif
  158. #ifdef CONFIG_ZONE_DMA32
  159. 256,
  160. #endif
  161. #ifdef CONFIG_HIGHMEM
  162. 32,
  163. #endif
  164. 32,
  165. };
  166. EXPORT_SYMBOL(totalram_pages);
  167. static char * const zone_names[MAX_NR_ZONES] = {
  168. #ifdef CONFIG_ZONE_DMA
  169. "DMA",
  170. #endif
  171. #ifdef CONFIG_ZONE_DMA32
  172. "DMA32",
  173. #endif
  174. "Normal",
  175. #ifdef CONFIG_HIGHMEM
  176. "HighMem",
  177. #endif
  178. "Movable",
  179. };
  180. int min_free_kbytes = 1024;
  181. static unsigned long __meminitdata nr_kernel_pages;
  182. static unsigned long __meminitdata nr_all_pages;
  183. static unsigned long __meminitdata dma_reserve;
  184. #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
  185. /* Movable memory ranges, will also be used by memblock subsystem. */
  186. struct movablemem_map movablemem_map;
  187. static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES];
  188. static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES];
  189. static unsigned long __initdata required_kernelcore;
  190. static unsigned long __initdata required_movablecore;
  191. static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES];
  192. static unsigned long __meminitdata zone_movable_limit[MAX_NUMNODES];
  193. /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
  194. int movable_zone;
  195. EXPORT_SYMBOL(movable_zone);
  196. #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
  197. #if MAX_NUMNODES > 1
  198. int nr_node_ids __read_mostly = MAX_NUMNODES;
  199. int nr_online_nodes __read_mostly = 1;
  200. EXPORT_SYMBOL(nr_node_ids);
  201. EXPORT_SYMBOL(nr_online_nodes);
  202. #endif
  203. int page_group_by_mobility_disabled __read_mostly;
  204. void set_pageblock_migratetype(struct page *page, int migratetype)
  205. {
  206. if (unlikely(page_group_by_mobility_disabled))
  207. migratetype = MIGRATE_UNMOVABLE;
  208. set_pageblock_flags_group(page, (unsigned long)migratetype,
  209. PB_migrate, PB_migrate_end);
  210. }
  211. bool oom_killer_disabled __read_mostly;
  212. #ifdef CONFIG_DEBUG_VM
  213. static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
  214. {
  215. int ret = 0;
  216. unsigned seq;
  217. unsigned long pfn = page_to_pfn(page);
  218. do {
  219. seq = zone_span_seqbegin(zone);
  220. if (pfn >= zone->zone_start_pfn + zone->spanned_pages)
  221. ret = 1;
  222. else if (pfn < zone->zone_start_pfn)
  223. ret = 1;
  224. } while (zone_span_seqretry(zone, seq));
  225. return ret;
  226. }
  227. static int page_is_consistent(struct zone *zone, struct page *page)
  228. {
  229. if (!pfn_valid_within(page_to_pfn(page)))
  230. return 0;
  231. if (zone != page_zone(page))
  232. return 0;
  233. return 1;
  234. }
  235. /*
  236. * Temporary debugging check for pages not lying within a given zone.
  237. */
  238. static int bad_range(struct zone *zone, struct page *page)
  239. {
  240. if (page_outside_zone_boundaries(zone, page))
  241. return 1;
  242. if (!page_is_consistent(zone, page))
  243. return 1;
  244. return 0;
  245. }
  246. #else
  247. static inline int bad_range(struct zone *zone, struct page *page)
  248. {
  249. return 0;
  250. }
  251. #endif
  252. static void bad_page(struct page *page)
  253. {
  254. static unsigned long resume;
  255. static unsigned long nr_shown;
  256. static unsigned long nr_unshown;
  257. /* Don't complain about poisoned pages */
  258. if (PageHWPoison(page)) {
  259. reset_page_mapcount(page); /* remove PageBuddy */
  260. return;
  261. }
  262. /*
  263. * Allow a burst of 60 reports, then keep quiet for that minute;
  264. * or allow a steady drip of one report per second.
  265. */
  266. if (nr_shown == 60) {
  267. if (time_before(jiffies, resume)) {
  268. nr_unshown++;
  269. goto out;
  270. }
  271. if (nr_unshown) {
  272. printk(KERN_ALERT
  273. "BUG: Bad page state: %lu messages suppressed\n",
  274. nr_unshown);
  275. nr_unshown = 0;
  276. }
  277. nr_shown = 0;
  278. }
  279. if (nr_shown++ == 0)
  280. resume = jiffies + 60 * HZ;
  281. printk(KERN_ALERT "BUG: Bad page state in process %s pfn:%05lx\n",
  282. current->comm, page_to_pfn(page));
  283. dump_page(page);
  284. print_modules();
  285. dump_stack();
  286. out:
  287. /* Leave bad fields for debug, except PageBuddy could make trouble */
  288. reset_page_mapcount(page); /* remove PageBuddy */
  289. add_taint(TAINT_BAD_PAGE);
  290. }
  291. /*
  292. * Higher-order pages are called "compound pages". They are structured thusly:
  293. *
  294. * The first PAGE_SIZE page is called the "head page".
  295. *
  296. * The remaining PAGE_SIZE pages are called "tail pages".
  297. *
  298. * All pages have PG_compound set. All tail pages have their ->first_page
  299. * pointing at the head page.
  300. *
  301. * The first tail page's ->lru.next holds the address of the compound page's
  302. * put_page() function. Its ->lru.prev holds the order of allocation.
  303. * This usage means that zero-order pages may not be compound.
  304. */
  305. static void free_compound_page(struct page *page)
  306. {
  307. __free_pages_ok(page, compound_order(page));
  308. }
  309. void prep_compound_page(struct page *page, unsigned long order)
  310. {
  311. int i;
  312. int nr_pages = 1 << order;
  313. set_compound_page_dtor(page, free_compound_page);
  314. set_compound_order(page, order);
  315. __SetPageHead(page);
  316. for (i = 1; i < nr_pages; i++) {
  317. struct page *p = page + i;
  318. __SetPageTail(p);
  319. set_page_count(p, 0);
  320. p->first_page = page;
  321. }
  322. }
  323. /* update __split_huge_page_refcount if you change this function */
  324. static int destroy_compound_page(struct page *page, unsigned long order)
  325. {
  326. int i;
  327. int nr_pages = 1 << order;
  328. int bad = 0;
  329. if (unlikely(compound_order(page) != order)) {
  330. bad_page(page);
  331. bad++;
  332. }
  333. __ClearPageHead(page);
  334. for (i = 1; i < nr_pages; i++) {
  335. struct page *p = page + i;
  336. if (unlikely(!PageTail(p) || (p->first_page != page))) {
  337. bad_page(page);
  338. bad++;
  339. }
  340. __ClearPageTail(p);
  341. }
  342. return bad;
  343. }
  344. static inline void prep_zero_page(struct page *page, int order, gfp_t gfp_flags)
  345. {
  346. int i;
  347. /*
  348. * clear_highpage() will use KM_USER0, so it's a bug to use __GFP_ZERO
  349. * and __GFP_HIGHMEM from hard or soft interrupt context.
  350. */
  351. VM_BUG_ON((gfp_flags & __GFP_HIGHMEM) && in_interrupt());
  352. for (i = 0; i < (1 << order); i++)
  353. clear_highpage(page + i);
  354. }
  355. #ifdef CONFIG_DEBUG_PAGEALLOC
  356. unsigned int _debug_guardpage_minorder;
  357. static int __init debug_guardpage_minorder_setup(char *buf)
  358. {
  359. unsigned long res;
  360. if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) {
  361. printk(KERN_ERR "Bad debug_guardpage_minorder value\n");
  362. return 0;
  363. }
  364. _debug_guardpage_minorder = res;
  365. printk(KERN_INFO "Setting debug_guardpage_minorder to %lu\n", res);
  366. return 0;
  367. }
  368. __setup("debug_guardpage_minorder=", debug_guardpage_minorder_setup);
  369. static inline void set_page_guard_flag(struct page *page)
  370. {
  371. __set_bit(PAGE_DEBUG_FLAG_GUARD, &page->debug_flags);
  372. }
  373. static inline void clear_page_guard_flag(struct page *page)
  374. {
  375. __clear_bit(PAGE_DEBUG_FLAG_GUARD, &page->debug_flags);
  376. }
  377. #else
  378. static inline void set_page_guard_flag(struct page *page) { }
  379. static inline void clear_page_guard_flag(struct page *page) { }
  380. #endif
  381. static inline void set_page_order(struct page *page, int order)
  382. {
  383. set_page_private(page, order);
  384. __SetPageBuddy(page);
  385. }
  386. static inline void rmv_page_order(struct page *page)
  387. {
  388. __ClearPageBuddy(page);
  389. set_page_private(page, 0);
  390. }
  391. /*
  392. * Locate the struct page for both the matching buddy in our
  393. * pair (buddy1) and the combined O(n+1) page they form (page).
  394. *
  395. * 1) Any buddy B1 will have an order O twin B2 which satisfies
  396. * the following equation:
  397. * B2 = B1 ^ (1 << O)
  398. * For example, if the starting buddy (buddy2) is #8 its order
  399. * 1 buddy is #10:
  400. * B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10
  401. *
  402. * 2) Any buddy B will have an order O+1 parent P which
  403. * satisfies the following equation:
  404. * P = B & ~(1 << O)
  405. *
  406. * Assumption: *_mem_map is contiguous at least up to MAX_ORDER
  407. */
  408. static inline unsigned long
  409. __find_buddy_index(unsigned long page_idx, unsigned int order)
  410. {
  411. return page_idx ^ (1 << order);
  412. }
  413. /*
  414. * This function checks whether a page is free && is the buddy
  415. * we can do coalesce a page and its buddy if
  416. * (a) the buddy is not in a hole &&
  417. * (b) the buddy is in the buddy system &&
  418. * (c) a page and its buddy have the same order &&
  419. * (d) a page and its buddy are in the same zone.
  420. *
  421. * For recording whether a page is in the buddy system, we set ->_mapcount -2.
  422. * Setting, clearing, and testing _mapcount -2 is serialized by zone->lock.
  423. *
  424. * For recording page's order, we use page_private(page).
  425. */
  426. static inline int page_is_buddy(struct page *page, struct page *buddy,
  427. int order)
  428. {
  429. if (!pfn_valid_within(page_to_pfn(buddy)))
  430. return 0;
  431. if (page_zone_id(page) != page_zone_id(buddy))
  432. return 0;
  433. if (page_is_guard(buddy) && page_order(buddy) == order) {
  434. VM_BUG_ON(page_count(buddy) != 0);
  435. return 1;
  436. }
  437. if (PageBuddy(buddy) && page_order(buddy) == order) {
  438. VM_BUG_ON(page_count(buddy) != 0);
  439. return 1;
  440. }
  441. return 0;
  442. }
  443. /*
  444. * Freeing function for a buddy system allocator.
  445. *
  446. * The concept of a buddy system is to maintain direct-mapped table
  447. * (containing bit values) for memory blocks of various "orders".
  448. * The bottom level table contains the map for the smallest allocatable
  449. * units of memory (here, pages), and each level above it describes
  450. * pairs of units from the levels below, hence, "buddies".
  451. * At a high level, all that happens here is marking the table entry
  452. * at the bottom level available, and propagating the changes upward
  453. * as necessary, plus some accounting needed to play nicely with other
  454. * parts of the VM system.
  455. * At each level, we keep a list of pages, which are heads of continuous
  456. * free pages of length of (1 << order) and marked with _mapcount -2. Page's
  457. * order is recorded in page_private(page) field.
  458. * So when we are allocating or freeing one, we can derive the state of the
  459. * other. That is, if we allocate a small block, and both were
  460. * free, the remainder of the region must be split into blocks.
  461. * If a block is freed, and its buddy is also free, then this
  462. * triggers coalescing into a block of larger size.
  463. *
  464. * -- nyc
  465. */
  466. static inline void __free_one_page(struct page *page,
  467. struct zone *zone, unsigned int order,
  468. int migratetype)
  469. {
  470. unsigned long page_idx;
  471. unsigned long combined_idx;
  472. unsigned long uninitialized_var(buddy_idx);
  473. struct page *buddy;
  474. if (unlikely(PageCompound(page)))
  475. if (unlikely(destroy_compound_page(page, order)))
  476. return;
  477. VM_BUG_ON(migratetype == -1);
  478. page_idx = page_to_pfn(page) & ((1 << MAX_ORDER) - 1);
  479. VM_BUG_ON(page_idx & ((1 << order) - 1));
  480. VM_BUG_ON(bad_range(zone, page));
  481. while (order < MAX_ORDER-1) {
  482. buddy_idx = __find_buddy_index(page_idx, order);
  483. buddy = page + (buddy_idx - page_idx);
  484. if (!page_is_buddy(page, buddy, order))
  485. break;
  486. /*
  487. * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
  488. * merge with it and move up one order.
  489. */
  490. if (page_is_guard(buddy)) {
  491. clear_page_guard_flag(buddy);
  492. set_page_private(page, 0);
  493. __mod_zone_freepage_state(zone, 1 << order,
  494. migratetype);
  495. } else {
  496. list_del(&buddy->lru);
  497. zone->free_area[order].nr_free--;
  498. rmv_page_order(buddy);
  499. }
  500. combined_idx = buddy_idx & page_idx;
  501. page = page + (combined_idx - page_idx);
  502. page_idx = combined_idx;
  503. order++;
  504. }
  505. set_page_order(page, order);
  506. /*
  507. * If this is not the largest possible page, check if the buddy
  508. * of the next-highest order is free. If it is, it's possible
  509. * that pages are being freed that will coalesce soon. In case,
  510. * that is happening, add the free page to the tail of the list
  511. * so it's less likely to be used soon and more likely to be merged
  512. * as a higher order page
  513. */
  514. if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {
  515. struct page *higher_page, *higher_buddy;
  516. combined_idx = buddy_idx & page_idx;
  517. higher_page = page + (combined_idx - page_idx);
  518. buddy_idx = __find_buddy_index(combined_idx, order + 1);
  519. higher_buddy = higher_page + (buddy_idx - combined_idx);
  520. if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
  521. list_add_tail(&page->lru,
  522. &zone->free_area[order].free_list[migratetype]);
  523. goto out;
  524. }
  525. }
  526. list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
  527. out:
  528. zone->free_area[order].nr_free++;
  529. }
  530. static inline int free_pages_check(struct page *page)
  531. {
  532. if (unlikely(page_mapcount(page) |
  533. (page->mapping != NULL) |
  534. (atomic_read(&page->_count) != 0) |
  535. (page->flags & PAGE_FLAGS_CHECK_AT_FREE) |
  536. (mem_cgroup_bad_page_check(page)))) {
  537. bad_page(page);
  538. return 1;
  539. }
  540. reset_page_last_nid(page);
  541. if (page->flags & PAGE_FLAGS_CHECK_AT_PREP)
  542. page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
  543. return 0;
  544. }
  545. /*
  546. * Frees a number of pages from the PCP lists
  547. * Assumes all pages on list are in same zone, and of same order.
  548. * count is the number of pages to free.
  549. *
  550. * If the zone was previously in an "all pages pinned" state then look to
  551. * see if this freeing clears that state.
  552. *
  553. * And clear the zone's pages_scanned counter, to hold off the "all pages are
  554. * pinned" detection logic.
  555. */
  556. static void free_pcppages_bulk(struct zone *zone, int count,
  557. struct per_cpu_pages *pcp)
  558. {
  559. int migratetype = 0;
  560. int batch_free = 0;
  561. int to_free = count;
  562. spin_lock(&zone->lock);
  563. zone->all_unreclaimable = 0;
  564. zone->pages_scanned = 0;
  565. while (to_free) {
  566. struct page *page;
  567. struct list_head *list;
  568. /*
  569. * Remove pages from lists in a round-robin fashion. A
  570. * batch_free count is maintained that is incremented when an
  571. * empty list is encountered. This is so more pages are freed
  572. * off fuller lists instead of spinning excessively around empty
  573. * lists
  574. */
  575. do {
  576. batch_free++;
  577. if (++migratetype == MIGRATE_PCPTYPES)
  578. migratetype = 0;
  579. list = &pcp->lists[migratetype];
  580. } while (list_empty(list));
  581. /* This is the only non-empty list. Free them all. */
  582. if (batch_free == MIGRATE_PCPTYPES)
  583. batch_free = to_free;
  584. do {
  585. int mt; /* migratetype of the to-be-freed page */
  586. page = list_entry(list->prev, struct page, lru);
  587. /* must delete as __free_one_page list manipulates */
  588. list_del(&page->lru);
  589. mt = get_freepage_migratetype(page);
  590. /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */
  591. __free_one_page(page, zone, 0, mt);
  592. trace_mm_page_pcpu_drain(page, 0, mt);
  593. if (likely(get_pageblock_migratetype(page) != MIGRATE_ISOLATE)) {
  594. __mod_zone_page_state(zone, NR_FREE_PAGES, 1);
  595. if (is_migrate_cma(mt))
  596. __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 1);
  597. }
  598. } while (--to_free && --batch_free && !list_empty(list));
  599. }
  600. spin_unlock(&zone->lock);
  601. }
  602. static void free_one_page(struct zone *zone, struct page *page, int order,
  603. int migratetype)
  604. {
  605. spin_lock(&zone->lock);
  606. zone->all_unreclaimable = 0;
  607. zone->pages_scanned = 0;
  608. __free_one_page(page, zone, order, migratetype);
  609. if (unlikely(migratetype != MIGRATE_ISOLATE))
  610. __mod_zone_freepage_state(zone, 1 << order, migratetype);
  611. spin_unlock(&zone->lock);
  612. }
  613. static bool free_pages_prepare(struct page *page, unsigned int order)
  614. {
  615. int i;
  616. int bad = 0;
  617. trace_mm_page_free(page, order);
  618. kmemcheck_free_shadow(page, order);
  619. if (PageAnon(page))
  620. page->mapping = NULL;
  621. for (i = 0; i < (1 << order); i++)
  622. bad += free_pages_check(page + i);
  623. if (bad)
  624. return false;
  625. if (!PageHighMem(page)) {
  626. debug_check_no_locks_freed(page_address(page),PAGE_SIZE<<order);
  627. debug_check_no_obj_freed(page_address(page),
  628. PAGE_SIZE << order);
  629. }
  630. arch_free_page(page, order);
  631. kernel_map_pages(page, 1 << order, 0);
  632. return true;
  633. }
  634. static void __free_pages_ok(struct page *page, unsigned int order)
  635. {
  636. unsigned long flags;
  637. int migratetype;
  638. if (!free_pages_prepare(page, order))
  639. return;
  640. local_irq_save(flags);
  641. __count_vm_events(PGFREE, 1 << order);
  642. migratetype = get_pageblock_migratetype(page);
  643. set_freepage_migratetype(page, migratetype);
  644. free_one_page(page_zone(page), page, order, migratetype);
  645. local_irq_restore(flags);
  646. }
  647. /*
  648. * Read access to zone->managed_pages is safe because it's unsigned long,
  649. * but we still need to serialize writers. Currently all callers of
  650. * __free_pages_bootmem() except put_page_bootmem() should only be used
  651. * at boot time. So for shorter boot time, we shift the burden to
  652. * put_page_bootmem() to serialize writers.
  653. */
  654. void __meminit __free_pages_bootmem(struct page *page, unsigned int order)
  655. {
  656. unsigned int nr_pages = 1 << order;
  657. unsigned int loop;
  658. prefetchw(page);
  659. for (loop = 0; loop < nr_pages; loop++) {
  660. struct page *p = &page[loop];
  661. if (loop + 1 < nr_pages)
  662. prefetchw(p + 1);
  663. __ClearPageReserved(p);
  664. set_page_count(p, 0);
  665. }
  666. page_zone(page)->managed_pages += 1 << order;
  667. set_page_refcounted(page);
  668. __free_pages(page, order);
  669. }
  670. #ifdef CONFIG_CMA
  671. /* Free whole pageblock and set it's migration type to MIGRATE_CMA. */
  672. void __init init_cma_reserved_pageblock(struct page *page)
  673. {
  674. unsigned i = pageblock_nr_pages;
  675. struct page *p = page;
  676. do {
  677. __ClearPageReserved(p);
  678. set_page_count(p, 0);
  679. } while (++p, --i);
  680. set_page_refcounted(page);
  681. set_pageblock_migratetype(page, MIGRATE_CMA);
  682. __free_pages(page, pageblock_order);
  683. totalram_pages += pageblock_nr_pages;
  684. #ifdef CONFIG_HIGHMEM
  685. if (PageHighMem(page))
  686. totalhigh_pages += pageblock_nr_pages;
  687. #endif
  688. }
  689. #endif
  690. /*
  691. * The order of subdivision here is critical for the IO subsystem.
  692. * Please do not alter this order without good reasons and regression
  693. * testing. Specifically, as large blocks of memory are subdivided,
  694. * the order in which smaller blocks are delivered depends on the order
  695. * they're subdivided in this function. This is the primary factor
  696. * influencing the order in which pages are delivered to the IO
  697. * subsystem according to empirical testing, and this is also justified
  698. * by considering the behavior of a buddy system containing a single
  699. * large block of memory acted on by a series of small allocations.
  700. * This behavior is a critical factor in sglist merging's success.
  701. *
  702. * -- nyc
  703. */
  704. static inline void expand(struct zone *zone, struct page *page,
  705. int low, int high, struct free_area *area,
  706. int migratetype)
  707. {
  708. unsigned long size = 1 << high;
  709. while (high > low) {
  710. area--;
  711. high--;
  712. size >>= 1;
  713. VM_BUG_ON(bad_range(zone, &page[size]));
  714. #ifdef CONFIG_DEBUG_PAGEALLOC
  715. if (high < debug_guardpage_minorder()) {
  716. /*
  717. * Mark as guard pages (or page), that will allow to
  718. * merge back to allocator when buddy will be freed.
  719. * Corresponding page table entries will not be touched,
  720. * pages will stay not present in virtual address space
  721. */
  722. INIT_LIST_HEAD(&page[size].lru);
  723. set_page_guard_flag(&page[size]);
  724. set_page_private(&page[size], high);
  725. /* Guard pages are not available for any usage */
  726. __mod_zone_freepage_state(zone, -(1 << high),
  727. migratetype);
  728. continue;
  729. }
  730. #endif
  731. list_add(&page[size].lru, &area->free_list[migratetype]);
  732. area->nr_free++;
  733. set_page_order(&page[size], high);
  734. }
  735. }
  736. /*
  737. * This page is about to be returned from the page allocator
  738. */
  739. static inline int check_new_page(struct page *page)
  740. {
  741. if (unlikely(page_mapcount(page) |
  742. (page->mapping != NULL) |
  743. (atomic_read(&page->_count) != 0) |
  744. (page->flags & PAGE_FLAGS_CHECK_AT_PREP) |
  745. (mem_cgroup_bad_page_check(page)))) {
  746. bad_page(page);
  747. return 1;
  748. }
  749. return 0;
  750. }
  751. static int prep_new_page(struct page *page, int order, gfp_t gfp_flags)
  752. {
  753. int i;
  754. for (i = 0; i < (1 << order); i++) {
  755. struct page *p = page + i;
  756. if (unlikely(check_new_page(p)))
  757. return 1;
  758. }
  759. set_page_private(page, 0);
  760. set_page_refcounted(page);
  761. arch_alloc_page(page, order);
  762. kernel_map_pages(page, 1 << order, 1);
  763. if (gfp_flags & __GFP_ZERO)
  764. prep_zero_page(page, order, gfp_flags);
  765. if (order && (gfp_flags & __GFP_COMP))
  766. prep_compound_page(page, order);
  767. return 0;
  768. }
  769. /*
  770. * Go through the free lists for the given migratetype and remove
  771. * the smallest available page from the freelists
  772. */
  773. static inline
  774. struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
  775. int migratetype)
  776. {
  777. unsigned int current_order;
  778. struct free_area * area;
  779. struct page *page;
  780. /* Find a page of the appropriate size in the preferred list */
  781. for (current_order = order; current_order < MAX_ORDER; ++current_order) {
  782. area = &(zone->free_area[current_order]);
  783. if (list_empty(&area->free_list[migratetype]))
  784. continue;
  785. page = list_entry(area->free_list[migratetype].next,
  786. struct page, lru);
  787. list_del(&page->lru);
  788. rmv_page_order(page);
  789. area->nr_free--;
  790. expand(zone, page, order, current_order, area, migratetype);
  791. return page;
  792. }
  793. return NULL;
  794. }
  795. /*
  796. * This array describes the order lists are fallen back to when
  797. * the free lists for the desirable migrate type are depleted
  798. */
  799. static int fallbacks[MIGRATE_TYPES][4] = {
  800. [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE },
  801. [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE },
  802. #ifdef CONFIG_CMA
  803. [MIGRATE_MOVABLE] = { MIGRATE_CMA, MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },
  804. [MIGRATE_CMA] = { MIGRATE_RESERVE }, /* Never used */
  805. #else
  806. [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },
  807. #endif
  808. [MIGRATE_RESERVE] = { MIGRATE_RESERVE }, /* Never used */
  809. [MIGRATE_ISOLATE] = { MIGRATE_RESERVE }, /* Never used */
  810. };
  811. /*
  812. * Move the free pages in a range to the free lists of the requested type.
  813. * Note that start_page and end_pages are not aligned on a pageblock
  814. * boundary. If alignment is required, use move_freepages_block()
  815. */
  816. int move_freepages(struct zone *zone,
  817. struct page *start_page, struct page *end_page,
  818. int migratetype)
  819. {
  820. struct page *page;
  821. unsigned long order;
  822. int pages_moved = 0;
  823. #ifndef CONFIG_HOLES_IN_ZONE
  824. /*
  825. * page_zone is not safe to call in this context when
  826. * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant
  827. * anyway as we check zone boundaries in move_freepages_block().
  828. * Remove at a later date when no bug reports exist related to
  829. * grouping pages by mobility
  830. */
  831. BUG_ON(page_zone(start_page) != page_zone(end_page));
  832. #endif
  833. for (page = start_page; page <= end_page;) {
  834. /* Make sure we are not inadvertently changing nodes */
  835. VM_BUG_ON(page_to_nid(page) != zone_to_nid(zone));
  836. if (!pfn_valid_within(page_to_pfn(page))) {
  837. page++;
  838. continue;
  839. }
  840. if (!PageBuddy(page)) {
  841. page++;
  842. continue;
  843. }
  844. order = page_order(page);
  845. list_move(&page->lru,
  846. &zone->free_area[order].free_list[migratetype]);
  847. set_freepage_migratetype(page, migratetype);
  848. page += 1 << order;
  849. pages_moved += 1 << order;
  850. }
  851. return pages_moved;
  852. }
  853. int move_freepages_block(struct zone *zone, struct page *page,
  854. int migratetype)
  855. {
  856. unsigned long start_pfn, end_pfn;
  857. struct page *start_page, *end_page;
  858. start_pfn = page_to_pfn(page);
  859. start_pfn = start_pfn & ~(pageblock_nr_pages-1);
  860. start_page = pfn_to_page(start_pfn);
  861. end_page = start_page + pageblock_nr_pages - 1;
  862. end_pfn = start_pfn + pageblock_nr_pages - 1;
  863. /* Do not cross zone boundaries */
  864. if (start_pfn < zone->zone_start_pfn)
  865. start_page = page;
  866. if (end_pfn >= zone->zone_start_pfn + zone->spanned_pages)
  867. return 0;
  868. return move_freepages(zone, start_page, end_page, migratetype);
  869. }
  870. static void change_pageblock_range(struct page *pageblock_page,
  871. int start_order, int migratetype)
  872. {
  873. int nr_pageblocks = 1 << (start_order - pageblock_order);
  874. while (nr_pageblocks--) {
  875. set_pageblock_migratetype(pageblock_page, migratetype);
  876. pageblock_page += pageblock_nr_pages;
  877. }
  878. }
  879. /* Remove an element from the buddy allocator from the fallback list */
  880. static inline struct page *
  881. __rmqueue_fallback(struct zone *zone, int order, int start_migratetype)
  882. {
  883. struct free_area * area;
  884. int current_order;
  885. struct page *page;
  886. int migratetype, i;
  887. /* Find the largest possible block of pages in the other list */
  888. for (current_order = MAX_ORDER-1; current_order >= order;
  889. --current_order) {
  890. for (i = 0;; i++) {
  891. migratetype = fallbacks[start_migratetype][i];
  892. /* MIGRATE_RESERVE handled later if necessary */
  893. if (migratetype == MIGRATE_RESERVE)
  894. break;
  895. area = &(zone->free_area[current_order]);
  896. if (list_empty(&area->free_list[migratetype]))
  897. continue;
  898. page = list_entry(area->free_list[migratetype].next,
  899. struct page, lru);
  900. area->nr_free--;
  901. /*
  902. * If breaking a large block of pages, move all free
  903. * pages to the preferred allocation list. If falling
  904. * back for a reclaimable kernel allocation, be more
  905. * aggressive about taking ownership of free pages
  906. *
  907. * On the other hand, never change migration
  908. * type of MIGRATE_CMA pageblocks nor move CMA
  909. * pages on different free lists. We don't
  910. * want unmovable pages to be allocated from
  911. * MIGRATE_CMA areas.
  912. */
  913. if (!is_migrate_cma(migratetype) &&
  914. (unlikely(current_order >= pageblock_order / 2) ||
  915. start_migratetype == MIGRATE_RECLAIMABLE ||
  916. page_group_by_mobility_disabled)) {
  917. int pages;
  918. pages = move_freepages_block(zone, page,
  919. start_migratetype);
  920. /* Claim the whole block if over half of it is free */
  921. if (pages >= (1 << (pageblock_order-1)) ||
  922. page_group_by_mobility_disabled)
  923. set_pageblock_migratetype(page,
  924. start_migratetype);
  925. migratetype = start_migratetype;
  926. }
  927. /* Remove the page from the freelists */
  928. list_del(&page->lru);
  929. rmv_page_order(page);
  930. /* Take ownership for orders >= pageblock_order */
  931. if (current_order >= pageblock_order &&
  932. !is_migrate_cma(migratetype))
  933. change_pageblock_range(page, current_order,
  934. start_migratetype);
  935. expand(zone, page, order, current_order, area,
  936. is_migrate_cma(migratetype)
  937. ? migratetype : start_migratetype);
  938. trace_mm_page_alloc_extfrag(page, order, current_order,
  939. start_migratetype, migratetype);
  940. return page;
  941. }
  942. }
  943. return NULL;
  944. }
  945. /*
  946. * Do the hard work of removing an element from the buddy allocator.
  947. * Call me with the zone->lock already held.
  948. */
  949. static struct page *__rmqueue(struct zone *zone, unsigned int order,
  950. int migratetype)
  951. {
  952. struct page *page;
  953. retry_reserve:
  954. page = __rmqueue_smallest(zone, order, migratetype);
  955. if (unlikely(!page) && migratetype != MIGRATE_RESERVE) {
  956. page = __rmqueue_fallback(zone, order, migratetype);
  957. /*
  958. * Use MIGRATE_RESERVE rather than fail an allocation. goto
  959. * is used because __rmqueue_smallest is an inline function
  960. * and we want just one call site
  961. */
  962. if (!page) {
  963. migratetype = MIGRATE_RESERVE;
  964. goto retry_reserve;
  965. }
  966. }
  967. trace_mm_page_alloc_zone_locked(page, order, migratetype);
  968. return page;
  969. }
  970. /*
  971. * Obtain a specified number of elements from the buddy allocator, all under
  972. * a single hold of the lock, for efficiency. Add them to the supplied list.
  973. * Returns the number of new pages which were placed at *list.
  974. */
  975. static int rmqueue_bulk(struct zone *zone, unsigned int order,
  976. unsigned long count, struct list_head *list,
  977. int migratetype, int cold)
  978. {
  979. int mt = migratetype, i;
  980. spin_lock(&zone->lock);
  981. for (i = 0; i < count; ++i) {
  982. struct page *page = __rmqueue(zone, order, migratetype);
  983. if (unlikely(page == NULL))
  984. break;
  985. /*
  986. * Split buddy pages returned by expand() are received here
  987. * in physical page order. The page is added to the callers and
  988. * list and the list head then moves forward. From the callers
  989. * perspective, the linked list is ordered by page number in
  990. * some conditions. This is useful for IO devices that can
  991. * merge IO requests if the physical pages are ordered
  992. * properly.
  993. */
  994. if (likely(cold == 0))
  995. list_add(&page->lru, list);
  996. else
  997. list_add_tail(&page->lru, list);
  998. if (IS_ENABLED(CONFIG_CMA)) {
  999. mt = get_pageblock_migratetype(page);
  1000. if (!is_migrate_cma(mt) && mt != MIGRATE_ISOLATE)
  1001. mt = migratetype;
  1002. }
  1003. set_freepage_migratetype(page, mt);
  1004. list = &page->lru;
  1005. if (is_migrate_cma(mt))
  1006. __mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
  1007. -(1 << order));
  1008. }
  1009. __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
  1010. spin_unlock(&zone->lock);
  1011. return i;
  1012. }
  1013. #ifdef CONFIG_NUMA
  1014. /*
  1015. * Called from the vmstat counter updater to drain pagesets of this
  1016. * currently executing processor on remote nodes after they have
  1017. * expired.
  1018. *
  1019. * Note that this function must be called with the thread pinned to
  1020. * a single processor.
  1021. */
  1022. void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
  1023. {
  1024. unsigned long flags;
  1025. int to_drain;
  1026. local_irq_save(flags);
  1027. if (pcp->count >= pcp->batch)
  1028. to_drain = pcp->batch;
  1029. else
  1030. to_drain = pcp->count;
  1031. if (to_drain > 0) {
  1032. free_pcppages_bulk(zone, to_drain, pcp);
  1033. pcp->count -= to_drain;
  1034. }
  1035. local_irq_restore(flags);
  1036. }
  1037. #endif
  1038. /*
  1039. * Drain pages of the indicated processor.
  1040. *
  1041. * The processor must either be the current processor and the
  1042. * thread pinned to the current processor or a processor that
  1043. * is not online.
  1044. */
  1045. static void drain_pages(unsigned int cpu)
  1046. {
  1047. unsigned long flags;
  1048. struct zone *zone;
  1049. for_each_populated_zone(zone) {
  1050. struct per_cpu_pageset *pset;
  1051. struct per_cpu_pages *pcp;
  1052. local_irq_save(flags);
  1053. pset = per_cpu_ptr(zone->pageset, cpu);
  1054. pcp = &pset->pcp;
  1055. if (pcp->count) {
  1056. free_pcppages_bulk(zone, pcp->count, pcp);
  1057. pcp->count = 0;
  1058. }
  1059. local_irq_restore(flags);
  1060. }
  1061. }
  1062. /*
  1063. * Spill all of this CPU's per-cpu pages back into the buddy allocator.
  1064. */
  1065. void drain_local_pages(void *arg)
  1066. {
  1067. drain_pages(smp_processor_id());
  1068. }
  1069. /*
  1070. * Spill all the per-cpu pages from all CPUs back into the buddy allocator.
  1071. *
  1072. * Note that this code is protected against sending an IPI to an offline
  1073. * CPU but does not guarantee sending an IPI to newly hotplugged CPUs:
  1074. * on_each_cpu_mask() blocks hotplug and won't talk to offlined CPUs but
  1075. * nothing keeps CPUs from showing up after we populated the cpumask and
  1076. * before the call to on_each_cpu_mask().
  1077. */
  1078. void drain_all_pages(void)
  1079. {
  1080. int cpu;
  1081. struct per_cpu_pageset *pcp;
  1082. struct zone *zone;
  1083. /*
  1084. * Allocate in the BSS so we wont require allocation in
  1085. * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y
  1086. */
  1087. static cpumask_t cpus_with_pcps;
  1088. /*
  1089. * We don't care about racing with CPU hotplug event
  1090. * as offline notification will cause the notified
  1091. * cpu to drain that CPU pcps and on_each_cpu_mask
  1092. * disables preemption as part of its processing
  1093. */
  1094. for_each_online_cpu(cpu) {
  1095. bool has_pcps = false;
  1096. for_each_populated_zone(zone) {
  1097. pcp = per_cpu_ptr(zone->pageset, cpu);
  1098. if (pcp->pcp.count) {
  1099. has_pcps = true;
  1100. break;
  1101. }
  1102. }
  1103. if (has_pcps)
  1104. cpumask_set_cpu(cpu, &cpus_with_pcps);
  1105. else
  1106. cpumask_clear_cpu(cpu, &cpus_with_pcps);
  1107. }
  1108. on_each_cpu_mask(&cpus_with_pcps, drain_local_pages, NULL, 1);
  1109. }
  1110. #ifdef CONFIG_HIBERNATION
  1111. void mark_free_pages(struct zone *zone)
  1112. {
  1113. unsigned long pfn, max_zone_pfn;
  1114. unsigned long flags;
  1115. int order, t;
  1116. struct list_head *curr;
  1117. if (!zone->spanned_pages)
  1118. return;
  1119. spin_lock_irqsave(&zone->lock, flags);
  1120. max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
  1121. for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
  1122. if (pfn_valid(pfn)) {
  1123. struct page *page = pfn_to_page(pfn);
  1124. if (!swsusp_page_is_forbidden(page))
  1125. swsusp_unset_page_free(page);
  1126. }
  1127. for_each_migratetype_order(order, t) {
  1128. list_for_each(curr, &zone->free_area[order].free_list[t]) {
  1129. unsigned long i;
  1130. pfn = page_to_pfn(list_entry(curr, struct page, lru));
  1131. for (i = 0; i < (1UL << order); i++)
  1132. swsusp_set_page_free(pfn_to_page(pfn + i));
  1133. }
  1134. }
  1135. spin_unlock_irqrestore(&zone->lock, flags);
  1136. }
  1137. #endif /* CONFIG_PM */
  1138. /*
  1139. * Free a 0-order page
  1140. * cold == 1 ? free a cold page : free a hot page
  1141. */
  1142. void free_hot_cold_page(struct page *page, int cold)
  1143. {
  1144. struct zone *zone = page_zone(page);
  1145. struct per_cpu_pages *pcp;
  1146. unsigned long flags;
  1147. int migratetype;
  1148. if (!free_pages_prepare(page, 0))
  1149. return;
  1150. migratetype = get_pageblock_migratetype(page);
  1151. set_freepage_migratetype(page, migratetype);
  1152. local_irq_save(flags);
  1153. __count_vm_event(PGFREE);
  1154. /*
  1155. * We only track unmovable, reclaimable and movable on pcp lists.
  1156. * Free ISOLATE pages back to the allocator because they are being
  1157. * offlined but treat RESERVE as movable pages so we can get those
  1158. * areas back if necessary. Otherwise, we may have to free
  1159. * excessively into the page allocator
  1160. */
  1161. if (migratetype >= MIGRATE_PCPTYPES) {
  1162. if (unlikely(migratetype == MIGRATE_ISOLATE)) {
  1163. free_one_page(zone, page, 0, migratetype);
  1164. goto out;
  1165. }
  1166. migratetype = MIGRATE_MOVABLE;
  1167. }
  1168. pcp = &this_cpu_ptr(zone->pageset)->pcp;
  1169. if (cold)
  1170. list_add_tail(&page->lru, &pcp->lists[migratetype]);
  1171. else
  1172. list_add(&page->lru, &pcp->lists[migratetype]);
  1173. pcp->count++;
  1174. if (pcp->count >= pcp->high) {
  1175. free_pcppages_bulk(zone, pcp->batch, pcp);
  1176. pcp->count -= pcp->batch;
  1177. }
  1178. out:
  1179. local_irq_restore(flags);
  1180. }
  1181. /*
  1182. * Free a list of 0-order pages
  1183. */
  1184. void free_hot_cold_page_list(struct list_head *list, int cold)
  1185. {
  1186. struct page *page, *next;
  1187. list_for_each_entry_safe(page, next, list, lru) {
  1188. trace_mm_page_free_batched(page, cold);
  1189. free_hot_cold_page(page, cold);
  1190. }
  1191. }
  1192. /*
  1193. * split_page takes a non-compound higher-order page, and splits it into
  1194. * n (1<<order) sub-pages: page[0..n]
  1195. * Each sub-page must be freed individually.
  1196. *
  1197. * Note: this is probably too low level an operation for use in drivers.
  1198. * Please consult with lkml before using this in your driver.
  1199. */
  1200. void split_page(struct page *page, unsigned int order)
  1201. {
  1202. int i;
  1203. VM_BUG_ON(PageCompound(page));
  1204. VM_BUG_ON(!page_count(page));
  1205. #ifdef CONFIG_KMEMCHECK
  1206. /*
  1207. * Split shadow pages too, because free(page[0]) would
  1208. * otherwise free the whole shadow.
  1209. */
  1210. if (kmemcheck_page_is_tracked(page))
  1211. split_page(virt_to_page(page[0].shadow), order);
  1212. #endif
  1213. for (i = 1; i < (1 << order); i++)
  1214. set_page_refcounted(page + i);
  1215. }
  1216. static int __isolate_free_page(struct page *page, unsigned int order)
  1217. {
  1218. unsigned long watermark;
  1219. struct zone *zone;
  1220. int mt;
  1221. BUG_ON(!PageBuddy(page));
  1222. zone = page_zone(page);
  1223. mt = get_pageblock_migratetype(page);
  1224. if (mt != MIGRATE_ISOLATE) {
  1225. /* Obey watermarks as if the page was being allocated */
  1226. watermark = low_wmark_pages(zone) + (1 << order);
  1227. if (!zone_watermark_ok(zone, 0, watermark, 0, 0))
  1228. return 0;
  1229. __mod_zone_freepage_state(zone, -(1UL << order), mt);
  1230. }
  1231. /* Remove page from free list */
  1232. list_del(&page->lru);
  1233. zone->free_area[order].nr_free--;
  1234. rmv_page_order(page);
  1235. /* Set the pageblock if the isolated page is at least a pageblock */
  1236. if (order >= pageblock_order - 1) {
  1237. struct page *endpage = page + (1 << order) - 1;
  1238. for (; page < endpage; page += pageblock_nr_pages) {
  1239. int mt = get_pageblock_migratetype(page);
  1240. if (mt != MIGRATE_ISOLATE && !is_migrate_cma(mt))
  1241. set_pageblock_migratetype(page,
  1242. MIGRATE_MOVABLE);
  1243. }
  1244. }
  1245. return 1UL << order;
  1246. }
  1247. /*
  1248. * Similar to split_page except the page is already free. As this is only
  1249. * being used for migration, the migratetype of the block also changes.
  1250. * As this is called with interrupts disabled, the caller is responsible
  1251. * for calling arch_alloc_page() and kernel_map_page() after interrupts
  1252. * are enabled.
  1253. *
  1254. * Note: this is probably too low level an operation for use in drivers.
  1255. * Please consult with lkml before using this in your driver.
  1256. */
  1257. int split_free_page(struct page *page)
  1258. {
  1259. unsigned int order;
  1260. int nr_pages;
  1261. order = page_order(page);
  1262. nr_pages = __isolate_free_page(page, order);
  1263. if (!nr_pages)
  1264. return 0;
  1265. /* Split into individual pages */
  1266. set_page_refcounted(page);
  1267. split_page(page, order);
  1268. return nr_pages;
  1269. }
  1270. /*
  1271. * Really, prep_compound_page() should be called from __rmqueue_bulk(). But
  1272. * we cheat by calling it from here, in the order > 0 path. Saves a branch
  1273. * or two.
  1274. */
  1275. static inline
  1276. struct page *buffered_rmqueue(struct zone *preferred_zone,
  1277. struct zone *zone, int order, gfp_t gfp_flags,
  1278. int migratetype)
  1279. {
  1280. unsigned long flags;
  1281. struct page *page;
  1282. int cold = !!(gfp_flags & __GFP_COLD);
  1283. again:
  1284. if (likely(order == 0)) {
  1285. struct per_cpu_pages *pcp;
  1286. struct list_head *list;
  1287. local_irq_save(flags);
  1288. pcp = &this_cpu_ptr(zone->pageset)->pcp;
  1289. list = &pcp->lists[migratetype];
  1290. if (list_empty(list)) {
  1291. pcp->count += rmqueue_bulk(zone, 0,
  1292. pcp->batch, list,
  1293. migratetype, cold);
  1294. if (unlikely(list_empty(list)))
  1295. goto failed;
  1296. }
  1297. if (cold)
  1298. page = list_entry(list->prev, struct page, lru);
  1299. else
  1300. page = list_entry(list->next, struct page, lru);
  1301. list_del(&page->lru);
  1302. pcp->count--;
  1303. } else {
  1304. if (unlikely(gfp_flags & __GFP_NOFAIL)) {
  1305. /*
  1306. * __GFP_NOFAIL is not to be used in new code.
  1307. *
  1308. * All __GFP_NOFAIL callers should be fixed so that they
  1309. * properly detect and handle allocation failures.
  1310. *
  1311. * We most definitely don't want callers attempting to
  1312. * allocate greater than order-1 page units with
  1313. * __GFP_NOFAIL.
  1314. */
  1315. WARN_ON_ONCE(order > 1);
  1316. }
  1317. spin_lock_irqsave(&zone->lock, flags);
  1318. page = __rmqueue(zone, order, migratetype);
  1319. spin_unlock(&zone->lock);
  1320. if (!page)
  1321. goto failed;
  1322. __mod_zone_freepage_state(zone, -(1 << order),
  1323. get_pageblock_migratetype(page));
  1324. }
  1325. __count_zone_vm_events(PGALLOC, zone, 1 << order);
  1326. zone_statistics(preferred_zone, zone, gfp_flags);
  1327. local_irq_restore(flags);
  1328. VM_BUG_ON(bad_range(zone, page));
  1329. if (prep_new_page(page, order, gfp_flags))
  1330. goto again;
  1331. return page;
  1332. failed:
  1333. local_irq_restore(flags);
  1334. return NULL;
  1335. }
  1336. #ifdef CONFIG_FAIL_PAGE_ALLOC
  1337. static struct {
  1338. struct fault_attr attr;
  1339. u32 ignore_gfp_highmem;
  1340. u32 ignore_gfp_wait;
  1341. u32 min_order;
  1342. } fail_page_alloc = {
  1343. .attr = FAULT_ATTR_INITIALIZER,
  1344. .ignore_gfp_wait = 1,
  1345. .ignore_gfp_highmem = 1,
  1346. .min_order = 1,
  1347. };
  1348. static int __init setup_fail_page_alloc(char *str)
  1349. {
  1350. return setup_fault_attr(&fail_page_alloc.attr, str);
  1351. }
  1352. __setup("fail_page_alloc=", setup_fail_page_alloc);
  1353. static bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
  1354. {
  1355. if (order < fail_page_alloc.min_order)
  1356. return false;
  1357. if (gfp_mask & __GFP_NOFAIL)
  1358. return false;
  1359. if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
  1360. return false;
  1361. if (fail_page_alloc.ignore_gfp_wait && (gfp_mask & __GFP_WAIT))
  1362. return false;
  1363. return should_fail(&fail_page_alloc.attr, 1 << order);
  1364. }
  1365. #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
  1366. static int __init fail_page_alloc_debugfs(void)
  1367. {
  1368. umode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
  1369. struct dentry *dir;
  1370. dir = fault_create_debugfs_attr("fail_page_alloc", NULL,
  1371. &fail_page_alloc.attr);
  1372. if (IS_ERR(dir))
  1373. return PTR_ERR(dir);
  1374. if (!debugfs_create_bool("ignore-gfp-wait", mode, dir,
  1375. &fail_page_alloc.ignore_gfp_wait))
  1376. goto fail;
  1377. if (!debugfs_create_bool("ignore-gfp-highmem", mode, dir,
  1378. &fail_page_alloc.ignore_gfp_highmem))
  1379. goto fail;
  1380. if (!debugfs_create_u32("min-order", mode, dir,
  1381. &fail_page_alloc.min_order))
  1382. goto fail;
  1383. return 0;
  1384. fail:
  1385. debugfs_remove_recursive(dir);
  1386. return -ENOMEM;
  1387. }
  1388. late_initcall(fail_page_alloc_debugfs);
  1389. #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
  1390. #else /* CONFIG_FAIL_PAGE_ALLOC */
  1391. static inline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
  1392. {
  1393. return false;
  1394. }
  1395. #endif /* CONFIG_FAIL_PAGE_ALLOC */
  1396. /*
  1397. * Return true if free pages are above 'mark'. This takes into account the order
  1398. * of the allocation.
  1399. */
  1400. static bool __zone_watermark_ok(struct zone *z, int order, unsigned long mark,
  1401. int classzone_idx, int alloc_flags, long free_pages)
  1402. {
  1403. /* free_pages my go negative - that's OK */
  1404. long min = mark;
  1405. long lowmem_reserve = z->lowmem_reserve[classzone_idx];
  1406. int o;
  1407. free_pages -= (1 << order) - 1;
  1408. if (alloc_flags & ALLOC_HIGH)
  1409. min -= min / 2;
  1410. if (alloc_flags & ALLOC_HARDER)
  1411. min -= min / 4;
  1412. #ifdef CONFIG_CMA
  1413. /* If allocation can't use CMA areas don't use free CMA pages */
  1414. if (!(alloc_flags & ALLOC_CMA))
  1415. free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES);
  1416. #endif
  1417. if (free_pages <= min + lowmem_reserve)
  1418. return false;
  1419. for (o = 0; o < order; o++) {
  1420. /* At the next order, this order's pages become unavailable */
  1421. free_pages -= z->free_area[o].nr_free << o;
  1422. /* Require fewer higher order pages to be free */
  1423. min >>= 1;
  1424. if (free_pages <= min)
  1425. return false;
  1426. }
  1427. return true;
  1428. }
  1429. bool zone_watermark_ok(struct zone *z, int order, unsigned long mark,
  1430. int classzone_idx, int alloc_flags)
  1431. {
  1432. return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
  1433. zone_page_state(z, NR_FREE_PAGES));
  1434. }
  1435. bool zone_watermark_ok_safe(struct zone *z, int order, unsigned long mark,
  1436. int classzone_idx, int alloc_flags)
  1437. {
  1438. long free_pages = zone_page_state(z, NR_FREE_PAGES);
  1439. if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
  1440. free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
  1441. return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
  1442. free_pages);
  1443. }
  1444. #ifdef CONFIG_NUMA
  1445. /*
  1446. * zlc_setup - Setup for "zonelist cache". Uses cached zone data to
  1447. * skip over zones that are not allowed by the cpuset, or that have
  1448. * been recently (in last second) found to be nearly full. See further
  1449. * comments in mmzone.h. Reduces cache footprint of zonelist scans
  1450. * that have to skip over a lot of full or unallowed zones.
  1451. *
  1452. * If the zonelist cache is present in the passed in zonelist, then
  1453. * returns a pointer to the allowed node mask (either the current
  1454. * tasks mems_allowed, or node_states[N_MEMORY].)
  1455. *
  1456. * If the zonelist cache is not available for this zonelist, does
  1457. * nothing and returns NULL.
  1458. *
  1459. * If the fullzones BITMAP in the zonelist cache is stale (more than
  1460. * a second since last zap'd) then we zap it out (clear its bits.)
  1461. *
  1462. * We hold off even calling zlc_setup, until after we've checked the
  1463. * first zone in the zonelist, on the theory that most allocations will
  1464. * be satisfied from that first zone, so best to examine that zone as
  1465. * quickly as we can.
  1466. */
  1467. static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
  1468. {
  1469. struct zonelist_cache *zlc; /* cached zonelist speedup info */
  1470. nodemask_t *allowednodes; /* zonelist_cache approximation */
  1471. zlc = zonelist->zlcache_ptr;
  1472. if (!zlc)
  1473. return NULL;
  1474. if (time_after(jiffies, zlc->last_full_zap + HZ)) {
  1475. bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
  1476. zlc->last_full_zap = jiffies;
  1477. }
  1478. allowednodes = !in_interrupt() && (alloc_flags & ALLOC_CPUSET) ?
  1479. &cpuset_current_mems_allowed :
  1480. &node_states[N_MEMORY];
  1481. return allowednodes;
  1482. }
  1483. /*
  1484. * Given 'z' scanning a zonelist, run a couple of quick checks to see
  1485. * if it is worth looking at further for free memory:
  1486. * 1) Check that the zone isn't thought to be full (doesn't have its
  1487. * bit set in the zonelist_cache fullzones BITMAP).
  1488. * 2) Check that the zones node (obtained from the zonelist_cache
  1489. * z_to_n[] mapping) is allowed in the passed in allowednodes mask.
  1490. * Return true (non-zero) if zone is worth looking at further, or
  1491. * else return false (zero) if it is not.
  1492. *
  1493. * This check -ignores- the distinction between various watermarks,
  1494. * such as GFP_HIGH, GFP_ATOMIC, PF_MEMALLOC, ... If a zone is
  1495. * found to be full for any variation of these watermarks, it will
  1496. * be considered full for up to one second by all requests, unless
  1497. * we are so low on memory on all allowed nodes that we are forced
  1498. * into the second scan of the zonelist.
  1499. *
  1500. * In the second scan we ignore this zonelist cache and exactly
  1501. * apply the watermarks to all zones, even it is slower to do so.
  1502. * We are low on memory in the second scan, and should leave no stone
  1503. * unturned looking for a free page.
  1504. */
  1505. static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z,
  1506. nodemask_t *allowednodes)
  1507. {
  1508. struct zonelist_cache *zlc; /* cached zonelist speedup info */
  1509. int i; /* index of *z in zonelist zones */
  1510. int n; /* node that zone *z is on */
  1511. zlc = zonelist->zlcache_ptr;
  1512. if (!zlc)
  1513. return 1;
  1514. i = z - zonelist->_zonerefs;
  1515. n = zlc->z_to_n[i];
  1516. /* This zone is worth trying if it is allowed but not full */
  1517. return node_isset(n, *allowednodes) && !test_bit(i, zlc->fullzones);
  1518. }
  1519. /*
  1520. * Given 'z' scanning a zonelist, set the corresponding bit in
  1521. * zlc->fullzones, so that subsequent attempts to allocate a page
  1522. * from that zone don't waste time re-examining it.
  1523. */
  1524. static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z)
  1525. {
  1526. struct zonelist_cache *zlc; /* cached zonelist speedup info */
  1527. int i; /* index of *z in zonelist zones */
  1528. zlc = zonelist->zlcache_ptr;
  1529. if (!zlc)
  1530. return;
  1531. i = z - zonelist->_zonerefs;
  1532. set_bit(i, zlc->fullzones);
  1533. }
  1534. /*
  1535. * clear all zones full, called after direct reclaim makes progress so that
  1536. * a zone that was recently full is not skipped over for up to a second
  1537. */
  1538. static void zlc_clear_zones_full(struct zonelist *zonelist)
  1539. {
  1540. struct zonelist_cache *zlc; /* cached zonelist speedup info */
  1541. zlc = zonelist->zlcache_ptr;
  1542. if (!zlc)
  1543. return;
  1544. bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
  1545. }
  1546. static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
  1547. {
  1548. return node_isset(local_zone->node, zone->zone_pgdat->reclaim_nodes);
  1549. }
  1550. static void __paginginit init_zone_allows_reclaim(int nid)
  1551. {
  1552. int i;
  1553. for_each_online_node(i)
  1554. if (node_distance(nid, i) <= RECLAIM_DISTANCE)
  1555. node_set(i, NODE_DATA(nid)->reclaim_nodes);
  1556. else
  1557. zone_reclaim_mode = 1;
  1558. }
  1559. #else /* CONFIG_NUMA */
  1560. static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
  1561. {
  1562. return NULL;
  1563. }
  1564. static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z,
  1565. nodemask_t *allowednodes)
  1566. {
  1567. return 1;
  1568. }
  1569. static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z)
  1570. {
  1571. }
  1572. static void zlc_clear_zones_full(struct zonelist *zonelist)
  1573. {
  1574. }
  1575. static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
  1576. {
  1577. return true;
  1578. }
  1579. static inline void init_zone_allows_reclaim(int nid)
  1580. {
  1581. }
  1582. #endif /* CONFIG_NUMA */
  1583. /*
  1584. * get_page_from_freelist goes through the zonelist trying to allocate
  1585. * a page.
  1586. */
  1587. static struct page *
  1588. get_page_from_freelist(gfp_t gfp_mask, nodemask_t *nodemask, unsigned int order,
  1589. struct zonelist *zonelist, int high_zoneidx, int alloc_flags,
  1590. struct zone *preferred_zone, int migratetype)
  1591. {
  1592. struct zoneref *z;
  1593. struct page *page = NULL;
  1594. int classzone_idx;
  1595. struct zone *zone;
  1596. nodemask_t *allowednodes = NULL;/* zonelist_cache approximation */
  1597. int zlc_active = 0; /* set if using zonelist_cache */
  1598. int did_zlc_setup = 0; /* just call zlc_setup() one time */
  1599. classzone_idx = zone_idx(preferred_zone);
  1600. zonelist_scan:
  1601. /*
  1602. * Scan zonelist, looking for a zone with enough free.
  1603. * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
  1604. */
  1605. for_each_zone_zonelist_nodemask(zone, z, zonelist,
  1606. high_zoneidx, nodemask) {
  1607. if (IS_ENABLED(CONFIG_NUMA) && zlc_active &&
  1608. !zlc_zone_worth_trying(zonelist, z, allowednodes))
  1609. continue;
  1610. if ((alloc_flags & ALLOC_CPUSET) &&
  1611. !cpuset_zone_allowed_softwall(zone, gfp_mask))
  1612. continue;
  1613. /*
  1614. * When allocating a page cache page for writing, we
  1615. * want to get it from a zone that is within its dirty
  1616. * limit, such that no single zone holds more than its
  1617. * proportional share of globally allowed dirty pages.
  1618. * The dirty limits take into account the zone's
  1619. * lowmem reserves and high watermark so that kswapd
  1620. * should be able to balance it without having to
  1621. * write pages from its LRU list.
  1622. *
  1623. * This may look like it could increase pressure on
  1624. * lower zones by failing allocations in higher zones
  1625. * before they are full. But the pages that do spill
  1626. * over are limited as the lower zones are protected
  1627. * by this very same mechanism. It should not become
  1628. * a practical burden to them.
  1629. *
  1630. * XXX: For now, allow allocations to potentially
  1631. * exceed the per-zone dirty limit in the slowpath
  1632. * (ALLOC_WMARK_LOW unset) before going into reclaim,
  1633. * which is important when on a NUMA setup the allowed
  1634. * zones are together not big enough to reach the
  1635. * global limit. The proper fix for these situations
  1636. * will require awareness of zones in the
  1637. * dirty-throttling and the flusher threads.
  1638. */
  1639. if ((alloc_flags & ALLOC_WMARK_LOW) &&
  1640. (gfp_mask & __GFP_WRITE) && !zone_dirty_ok(zone))
  1641. goto this_zone_full;
  1642. BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
  1643. if (!(alloc_flags & ALLOC_NO_WATERMARKS)) {
  1644. unsigned long mark;
  1645. int ret;
  1646. mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK];
  1647. if (zone_watermark_ok(zone, order, mark,
  1648. classzone_idx, alloc_flags))
  1649. goto try_this_zone;
  1650. if (IS_ENABLED(CONFIG_NUMA) &&
  1651. !did_zlc_setup && nr_online_nodes > 1) {
  1652. /*
  1653. * we do zlc_setup if there are multiple nodes
  1654. * and before considering the first zone allowed
  1655. * by the cpuset.
  1656. */
  1657. allowednodes = zlc_setup(zonelist, alloc_flags);
  1658. zlc_active = 1;
  1659. did_zlc_setup = 1;
  1660. }
  1661. if (zone_reclaim_mode == 0 ||
  1662. !zone_allows_reclaim(preferred_zone, zone))
  1663. goto this_zone_full;
  1664. /*
  1665. * As we may have just activated ZLC, check if the first
  1666. * eligible zone has failed zone_reclaim recently.
  1667. */
  1668. if (IS_ENABLED(CONFIG_NUMA) && zlc_active &&
  1669. !zlc_zone_worth_trying(zonelist, z, allowednodes))
  1670. continue;
  1671. ret = zone_reclaim(zone, gfp_mask, order);
  1672. switch (ret) {
  1673. case ZONE_RECLAIM_NOSCAN:
  1674. /* did not scan */
  1675. continue;
  1676. case ZONE_RECLAIM_FULL:
  1677. /* scanned but unreclaimable */
  1678. continue;
  1679. default:
  1680. /* did we reclaim enough */
  1681. if (!zone_watermark_ok(zone, order, mark,
  1682. classzone_idx, alloc_flags))
  1683. goto this_zone_full;
  1684. }
  1685. }
  1686. try_this_zone:
  1687. page = buffered_rmqueue(preferred_zone, zone, order,
  1688. gfp_mask, migratetype);
  1689. if (page)
  1690. break;
  1691. this_zone_full:
  1692. if (IS_ENABLED(CONFIG_NUMA))
  1693. zlc_mark_zone_full(zonelist, z);
  1694. }
  1695. if (unlikely(IS_ENABLED(CONFIG_NUMA) && page == NULL && zlc_active)) {
  1696. /* Disable zlc cache for second zonelist scan */
  1697. zlc_active = 0;
  1698. goto zonelist_scan;
  1699. }
  1700. if (page)
  1701. /*
  1702. * page->pfmemalloc is set when ALLOC_NO_WATERMARKS was
  1703. * necessary to allocate the page. The expectation is
  1704. * that the caller is taking steps that will free more
  1705. * memory. The caller should avoid the page being used
  1706. * for !PFMEMALLOC purposes.
  1707. */
  1708. page->pfmemalloc = !!(alloc_flags & ALLOC_NO_WATERMARKS);
  1709. return page;
  1710. }
  1711. /*
  1712. * Large machines with many possible nodes should not always dump per-node
  1713. * meminfo in irq context.
  1714. */
  1715. static inline bool should_suppress_show_mem(void)
  1716. {
  1717. bool ret = false;
  1718. #if NODES_SHIFT > 8
  1719. ret = in_interrupt();
  1720. #endif
  1721. return ret;
  1722. }
  1723. static DEFINE_RATELIMIT_STATE(nopage_rs,
  1724. DEFAULT_RATELIMIT_INTERVAL,
  1725. DEFAULT_RATELIMIT_BURST);
  1726. void warn_alloc_failed(gfp_t gfp_mask, int order, const char *fmt, ...)
  1727. {
  1728. unsigned int filter = SHOW_MEM_FILTER_NODES;
  1729. if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) ||
  1730. debug_guardpage_minorder() > 0)
  1731. return;
  1732. /*
  1733. * This documents exceptions given to allocations in certain
  1734. * contexts that are allowed to allocate outside current's set
  1735. * of allowed nodes.
  1736. */
  1737. if (!(gfp_mask & __GFP_NOMEMALLOC))
  1738. if (test_thread_flag(TIF_MEMDIE) ||
  1739. (current->flags & (PF_MEMALLOC | PF_EXITING)))
  1740. filter &= ~SHOW_MEM_FILTER_NODES;
  1741. if (in_interrupt() || !(gfp_mask & __GFP_WAIT))
  1742. filter &= ~SHOW_MEM_FILTER_NODES;
  1743. if (fmt) {
  1744. struct va_format vaf;
  1745. va_list args;
  1746. va_start(args, fmt);
  1747. vaf.fmt = fmt;
  1748. vaf.va = &args;
  1749. pr_warn("%pV", &vaf);
  1750. va_end(args);
  1751. }
  1752. pr_warn("%s: page allocation failure: order:%d, mode:0x%x\n",
  1753. current->comm, order, gfp_mask);
  1754. dump_stack();
  1755. if (!should_suppress_show_mem())
  1756. show_mem(filter);
  1757. }
  1758. static inline int
  1759. should_alloc_retry(gfp_t gfp_mask, unsigned int order,
  1760. unsigned long did_some_progress,
  1761. unsigned long pages_reclaimed)
  1762. {
  1763. /* Do not loop if specifically requested */
  1764. if (gfp_mask & __GFP_NORETRY)
  1765. return 0;
  1766. /* Always retry if specifically requested */
  1767. if (gfp_mask & __GFP_NOFAIL)
  1768. return 1;
  1769. /*
  1770. * Suspend converts GFP_KERNEL to __GFP_WAIT which can prevent reclaim
  1771. * making forward progress without invoking OOM. Suspend also disables
  1772. * storage devices so kswapd will not help. Bail if we are suspending.
  1773. */
  1774. if (!did_some_progress && pm_suspended_storage())
  1775. return 0;
  1776. /*
  1777. * In this implementation, order <= PAGE_ALLOC_COSTLY_ORDER
  1778. * means __GFP_NOFAIL, but that may not be true in other
  1779. * implementations.
  1780. */
  1781. if (order <= PAGE_ALLOC_COSTLY_ORDER)
  1782. return 1;
  1783. /*
  1784. * For order > PAGE_ALLOC_COSTLY_ORDER, if __GFP_REPEAT is
  1785. * specified, then we retry until we no longer reclaim any pages
  1786. * (above), or we've reclaimed an order of pages at least as
  1787. * large as the allocation's order. In both cases, if the
  1788. * allocation still fails, we stop retrying.
  1789. */
  1790. if (gfp_mask & __GFP_REPEAT && pages_reclaimed < (1 << order))
  1791. return 1;
  1792. return 0;
  1793. }
  1794. static inline struct page *
  1795. __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
  1796. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1797. nodemask_t *nodemask, struct zone *preferred_zone,
  1798. int migratetype)
  1799. {
  1800. struct page *page;
  1801. /* Acquire the OOM killer lock for the zones in zonelist */
  1802. if (!try_set_zonelist_oom(zonelist, gfp_mask)) {
  1803. schedule_timeout_uninterruptible(1);
  1804. return NULL;
  1805. }
  1806. /*
  1807. * Go through the zonelist yet one more time, keep very high watermark
  1808. * here, this is only to catch a parallel oom killing, we must fail if
  1809. * we're still under heavy pressure.
  1810. */
  1811. page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask,
  1812. order, zonelist, high_zoneidx,
  1813. ALLOC_WMARK_HIGH|ALLOC_CPUSET,
  1814. preferred_zone, migratetype);
  1815. if (page)
  1816. goto out;
  1817. if (!(gfp_mask & __GFP_NOFAIL)) {
  1818. /* The OOM killer will not help higher order allocs */
  1819. if (order > PAGE_ALLOC_COSTLY_ORDER)
  1820. goto out;
  1821. /* The OOM killer does not needlessly kill tasks for lowmem */
  1822. if (high_zoneidx < ZONE_NORMAL)
  1823. goto out;
  1824. /*
  1825. * GFP_THISNODE contains __GFP_NORETRY and we never hit this.
  1826. * Sanity check for bare calls of __GFP_THISNODE, not real OOM.
  1827. * The caller should handle page allocation failure by itself if
  1828. * it specifies __GFP_THISNODE.
  1829. * Note: Hugepage uses it but will hit PAGE_ALLOC_COSTLY_ORDER.
  1830. */
  1831. if (gfp_mask & __GFP_THISNODE)
  1832. goto out;
  1833. }
  1834. /* Exhausted what can be done so it's blamo time */
  1835. out_of_memory(zonelist, gfp_mask, order, nodemask, false);
  1836. out:
  1837. clear_zonelist_oom(zonelist, gfp_mask);
  1838. return page;
  1839. }
  1840. #ifdef CONFIG_COMPACTION
  1841. /* Try memory compaction for high-order allocations before reclaim */
  1842. static struct page *
  1843. __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
  1844. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1845. nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
  1846. int migratetype, bool sync_migration,
  1847. bool *contended_compaction, bool *deferred_compaction,
  1848. unsigned long *did_some_progress)
  1849. {
  1850. if (!order)
  1851. return NULL;
  1852. if (compaction_deferred(preferred_zone, order)) {
  1853. *deferred_compaction = true;
  1854. return NULL;
  1855. }
  1856. current->flags |= PF_MEMALLOC;
  1857. *did_some_progress = try_to_compact_pages(zonelist, order, gfp_mask,
  1858. nodemask, sync_migration,
  1859. contended_compaction);
  1860. current->flags &= ~PF_MEMALLOC;
  1861. if (*did_some_progress != COMPACT_SKIPPED) {
  1862. struct page *page;
  1863. /* Page migration frees to the PCP lists but we want merging */
  1864. drain_pages(get_cpu());
  1865. put_cpu();
  1866. page = get_page_from_freelist(gfp_mask, nodemask,
  1867. order, zonelist, high_zoneidx,
  1868. alloc_flags & ~ALLOC_NO_WATERMARKS,
  1869. preferred_zone, migratetype);
  1870. if (page) {
  1871. preferred_zone->compact_blockskip_flush = false;
  1872. preferred_zone->compact_considered = 0;
  1873. preferred_zone->compact_defer_shift = 0;
  1874. if (order >= preferred_zone->compact_order_failed)
  1875. preferred_zone->compact_order_failed = order + 1;
  1876. count_vm_event(COMPACTSUCCESS);
  1877. return page;
  1878. }
  1879. /*
  1880. * It's bad if compaction run occurs and fails.
  1881. * The most likely reason is that pages exist,
  1882. * but not enough to satisfy watermarks.
  1883. */
  1884. count_vm_event(COMPACTFAIL);
  1885. /*
  1886. * As async compaction considers a subset of pageblocks, only
  1887. * defer if the failure was a sync compaction failure.
  1888. */
  1889. if (sync_migration)
  1890. defer_compaction(preferred_zone, order);
  1891. cond_resched();
  1892. }
  1893. return NULL;
  1894. }
  1895. #else
  1896. static inline struct page *
  1897. __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
  1898. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1899. nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
  1900. int migratetype, bool sync_migration,
  1901. bool *contended_compaction, bool *deferred_compaction,
  1902. unsigned long *did_some_progress)
  1903. {
  1904. return NULL;
  1905. }
  1906. #endif /* CONFIG_COMPACTION */
  1907. /* Perform direct synchronous page reclaim */
  1908. static int
  1909. __perform_reclaim(gfp_t gfp_mask, unsigned int order, struct zonelist *zonelist,
  1910. nodemask_t *nodemask)
  1911. {
  1912. struct reclaim_state reclaim_state;
  1913. int progress;
  1914. cond_resched();
  1915. /* We now go into synchronous reclaim */
  1916. cpuset_memory_pressure_bump();
  1917. current->flags |= PF_MEMALLOC;
  1918. lockdep_set_current_reclaim_state(gfp_mask);
  1919. reclaim_state.reclaimed_slab = 0;
  1920. current->reclaim_state = &reclaim_state;
  1921. progress = try_to_free_pages(zonelist, order, gfp_mask, nodemask);
  1922. current->reclaim_state = NULL;
  1923. lockdep_clear_current_reclaim_state();
  1924. current->flags &= ~PF_MEMALLOC;
  1925. cond_resched();
  1926. return progress;
  1927. }
  1928. /* The really slow allocator path where we enter direct reclaim */
  1929. static inline struct page *
  1930. __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
  1931. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1932. nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
  1933. int migratetype, unsigned long *did_some_progress)
  1934. {
  1935. struct page *page = NULL;
  1936. bool drained = false;
  1937. *did_some_progress = __perform_reclaim(gfp_mask, order, zonelist,
  1938. nodemask);
  1939. if (unlikely(!(*did_some_progress)))
  1940. return NULL;
  1941. /* After successful reclaim, reconsider all zones for allocation */
  1942. if (IS_ENABLED(CONFIG_NUMA))
  1943. zlc_clear_zones_full(zonelist);
  1944. retry:
  1945. page = get_page_from_freelist(gfp_mask, nodemask, order,
  1946. zonelist, high_zoneidx,
  1947. alloc_flags & ~ALLOC_NO_WATERMARKS,
  1948. preferred_zone, migratetype);
  1949. /*
  1950. * If an allocation failed after direct reclaim, it could be because
  1951. * pages are pinned on the per-cpu lists. Drain them and try again
  1952. */
  1953. if (!page && !drained) {
  1954. drain_all_pages();
  1955. drained = true;
  1956. goto retry;
  1957. }
  1958. return page;
  1959. }
  1960. /*
  1961. * This is called in the allocator slow-path if the allocation request is of
  1962. * sufficient urgency to ignore watermarks and take other desperate measures
  1963. */
  1964. static inline struct page *
  1965. __alloc_pages_high_priority(gfp_t gfp_mask, unsigned int order,
  1966. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1967. nodemask_t *nodemask, struct zone *preferred_zone,
  1968. int migratetype)
  1969. {
  1970. struct page *page;
  1971. do {
  1972. page = get_page_from_freelist(gfp_mask, nodemask, order,
  1973. zonelist, high_zoneidx, ALLOC_NO_WATERMARKS,
  1974. preferred_zone, migratetype);
  1975. if (!page && gfp_mask & __GFP_NOFAIL)
  1976. wait_iff_congested(preferred_zone, BLK_RW_ASYNC, HZ/50);
  1977. } while (!page && (gfp_mask & __GFP_NOFAIL));
  1978. return page;
  1979. }
  1980. static inline
  1981. void wake_all_kswapd(unsigned int order, struct zonelist *zonelist,
  1982. enum zone_type high_zoneidx,
  1983. enum zone_type classzone_idx)
  1984. {
  1985. struct zoneref *z;
  1986. struct zone *zone;
  1987. for_each_zone_zonelist(zone, z, zonelist, high_zoneidx)
  1988. wakeup_kswapd(zone, order, classzone_idx);
  1989. }
  1990. static inline int
  1991. gfp_to_alloc_flags(gfp_t gfp_mask)
  1992. {
  1993. int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
  1994. const gfp_t wait = gfp_mask & __GFP_WAIT;
  1995. /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */
  1996. BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);
  1997. /*
  1998. * The caller may dip into page reserves a bit more if the caller
  1999. * cannot run direct reclaim, or if the caller has realtime scheduling
  2000. * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will
  2001. * set both ALLOC_HARDER (!wait) and ALLOC_HIGH (__GFP_HIGH).
  2002. */
  2003. alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH);
  2004. if (!wait) {
  2005. /*
  2006. * Not worth trying to allocate harder for
  2007. * __GFP_NOMEMALLOC even if it can't schedule.
  2008. */
  2009. if (!(gfp_mask & __GFP_NOMEMALLOC))
  2010. alloc_flags |= ALLOC_HARDER;
  2011. /*
  2012. * Ignore cpuset if GFP_ATOMIC (!wait) rather than fail alloc.
  2013. * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
  2014. */
  2015. alloc_flags &= ~ALLOC_CPUSET;
  2016. } else if (unlikely(rt_task(current)) && !in_interrupt())
  2017. alloc_flags |= ALLOC_HARDER;
  2018. if (likely(!(gfp_mask & __GFP_NOMEMALLOC))) {
  2019. if (gfp_mask & __GFP_MEMALLOC)
  2020. alloc_flags |= ALLOC_NO_WATERMARKS;
  2021. else if (in_serving_softirq() && (current->flags & PF_MEMALLOC))
  2022. alloc_flags |= ALLOC_NO_WATERMARKS;
  2023. else if (!in_interrupt() &&
  2024. ((current->flags & PF_MEMALLOC) ||
  2025. unlikely(test_thread_flag(TIF_MEMDIE))))
  2026. alloc_flags |= ALLOC_NO_WATERMARKS;
  2027. }
  2028. #ifdef CONFIG_CMA
  2029. if (allocflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
  2030. alloc_flags |= ALLOC_CMA;
  2031. #endif
  2032. return alloc_flags;
  2033. }
  2034. bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
  2035. {
  2036. return !!(gfp_to_alloc_flags(gfp_mask) & ALLOC_NO_WATERMARKS);
  2037. }
  2038. static inline struct page *
  2039. __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
  2040. struct zonelist *zonelist, enum zone_type high_zoneidx,
  2041. nodemask_t *nodemask, struct zone *preferred_zone,
  2042. int migratetype)
  2043. {
  2044. const gfp_t wait = gfp_mask & __GFP_WAIT;
  2045. struct page *page = NULL;
  2046. int alloc_flags;
  2047. unsigned long pages_reclaimed = 0;
  2048. unsigned long did_some_progress;
  2049. bool sync_migration = false;
  2050. bool deferred_compaction = false;
  2051. bool contended_compaction = false;
  2052. /*
  2053. * In the slowpath, we sanity check order to avoid ever trying to
  2054. * reclaim >= MAX_ORDER areas which will never succeed. Callers may
  2055. * be using allocators in order of preference for an area that is
  2056. * too large.
  2057. */
  2058. if (order >= MAX_ORDER) {
  2059. WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));
  2060. return NULL;
  2061. }
  2062. /*
  2063. * GFP_THISNODE (meaning __GFP_THISNODE, __GFP_NORETRY and
  2064. * __GFP_NOWARN set) should not cause reclaim since the subsystem
  2065. * (f.e. slab) using GFP_THISNODE may choose to trigger reclaim
  2066. * using a larger set of nodes after it has established that the
  2067. * allowed per node queues are empty and that nodes are
  2068. * over allocated.
  2069. */
  2070. if (IS_ENABLED(CONFIG_NUMA) &&
  2071. (gfp_mask & GFP_THISNODE) == GFP_THISNODE)
  2072. goto nopage;
  2073. restart:
  2074. if (!(gfp_mask & __GFP_NO_KSWAPD))
  2075. wake_all_kswapd(order, zonelist, high_zoneidx,
  2076. zone_idx(preferred_zone));
  2077. /*
  2078. * OK, we're below the kswapd watermark and have kicked background
  2079. * reclaim. Now things get more complex, so set up alloc_flags according
  2080. * to how we want to proceed.
  2081. */
  2082. alloc_flags = gfp_to_alloc_flags(gfp_mask);
  2083. /*
  2084. * Find the true preferred zone if the allocation is unconstrained by
  2085. * cpusets.
  2086. */
  2087. if (!(alloc_flags & ALLOC_CPUSET) && !nodemask)
  2088. first_zones_zonelist(zonelist, high_zoneidx, NULL,
  2089. &preferred_zone);
  2090. rebalance:
  2091. /* This is the last chance, in general, before the goto nopage. */
  2092. page = get_page_from_freelist(gfp_mask, nodemask, order, zonelist,
  2093. high_zoneidx, alloc_flags & ~ALLOC_NO_WATERMARKS,
  2094. preferred_zone, migratetype);
  2095. if (page)
  2096. goto got_pg;
  2097. /* Allocate without watermarks if the context allows */
  2098. if (alloc_flags & ALLOC_NO_WATERMARKS) {
  2099. /*
  2100. * Ignore mempolicies if ALLOC_NO_WATERMARKS on the grounds
  2101. * the allocation is high priority and these type of
  2102. * allocations are system rather than user orientated
  2103. */
  2104. zonelist = node_zonelist(numa_node_id(), gfp_mask);
  2105. page = __alloc_pages_high_priority(gfp_mask, order,
  2106. zonelist, high_zoneidx, nodemask,
  2107. preferred_zone, migratetype);
  2108. if (page) {
  2109. goto got_pg;
  2110. }
  2111. }
  2112. /* Atomic allocations - we can't balance anything */
  2113. if (!wait)
  2114. goto nopage;
  2115. /* Avoid recursion of direct reclaim */
  2116. if (current->flags & PF_MEMALLOC)
  2117. goto nopage;
  2118. /* Avoid allocations with no watermarks from looping endlessly */
  2119. if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL))
  2120. goto nopage;
  2121. /*
  2122. * Try direct compaction. The first pass is asynchronous. Subsequent
  2123. * attempts after direct reclaim are synchronous
  2124. */
  2125. page = __alloc_pages_direct_compact(gfp_mask, order,
  2126. zonelist, high_zoneidx,
  2127. nodemask,
  2128. alloc_flags, preferred_zone,
  2129. migratetype, sync_migration,
  2130. &contended_compaction,
  2131. &deferred_compaction,
  2132. &did_some_progress);
  2133. if (page)
  2134. goto got_pg;
  2135. sync_migration = true;
  2136. /*
  2137. * If compaction is deferred for high-order allocations, it is because
  2138. * sync compaction recently failed. In this is the case and the caller
  2139. * requested a movable allocation that does not heavily disrupt the
  2140. * system then fail the allocation instead of entering direct reclaim.
  2141. */
  2142. if ((deferred_compaction || contended_compaction) &&
  2143. (gfp_mask & __GFP_NO_KSWAPD))
  2144. goto nopage;
  2145. /* Try direct reclaim and then allocating */
  2146. page = __alloc_pages_direct_reclaim(gfp_mask, order,
  2147. zonelist, high_zoneidx,
  2148. nodemask,
  2149. alloc_flags, preferred_zone,
  2150. migratetype, &did_some_progress);
  2151. if (page)
  2152. goto got_pg;
  2153. /*
  2154. * If we failed to make any progress reclaiming, then we are
  2155. * running out of options and have to consider going OOM
  2156. */
  2157. if (!did_some_progress) {
  2158. if ((gfp_mask & __GFP_FS) && !(gfp_mask & __GFP_NORETRY)) {
  2159. if (oom_killer_disabled)
  2160. goto nopage;
  2161. /* Coredumps can quickly deplete all memory reserves */
  2162. if ((current->flags & PF_DUMPCORE) &&
  2163. !(gfp_mask & __GFP_NOFAIL))
  2164. goto nopage;
  2165. page = __alloc_pages_may_oom(gfp_mask, order,
  2166. zonelist, high_zoneidx,
  2167. nodemask, preferred_zone,
  2168. migratetype);
  2169. if (page)
  2170. goto got_pg;
  2171. if (!(gfp_mask & __GFP_NOFAIL)) {
  2172. /*
  2173. * The oom killer is not called for high-order
  2174. * allocations that may fail, so if no progress
  2175. * is being made, there are no other options and
  2176. * retrying is unlikely to help.
  2177. */
  2178. if (order > PAGE_ALLOC_COSTLY_ORDER)
  2179. goto nopage;
  2180. /*
  2181. * The oom killer is not called for lowmem
  2182. * allocations to prevent needlessly killing
  2183. * innocent tasks.
  2184. */
  2185. if (high_zoneidx < ZONE_NORMAL)
  2186. goto nopage;
  2187. }
  2188. goto restart;
  2189. }
  2190. }
  2191. /* Check if we should retry the allocation */
  2192. pages_reclaimed += did_some_progress;
  2193. if (should_alloc_retry(gfp_mask, order, did_some_progress,
  2194. pages_reclaimed)) {
  2195. /* Wait for some write requests to complete then retry */
  2196. wait_iff_congested(preferred_zone, BLK_RW_ASYNC, HZ/50);
  2197. goto rebalance;
  2198. } else {
  2199. /*
  2200. * High-order allocations do not necessarily loop after
  2201. * direct reclaim and reclaim/compaction depends on compaction
  2202. * being called after reclaim so call directly if necessary
  2203. */
  2204. page = __alloc_pages_direct_compact(gfp_mask, order,
  2205. zonelist, high_zoneidx,
  2206. nodemask,
  2207. alloc_flags, preferred_zone,
  2208. migratetype, sync_migration,
  2209. &contended_compaction,
  2210. &deferred_compaction,
  2211. &did_some_progress);
  2212. if (page)
  2213. goto got_pg;
  2214. }
  2215. nopage:
  2216. warn_alloc_failed(gfp_mask, order, NULL);
  2217. return page;
  2218. got_pg:
  2219. if (kmemcheck_enabled)
  2220. kmemcheck_pagealloc_alloc(page, order, gfp_mask);
  2221. return page;
  2222. }
  2223. /*
  2224. * This is the 'heart' of the zoned buddy allocator.
  2225. */
  2226. struct page *
  2227. __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
  2228. struct zonelist *zonelist, nodemask_t *nodemask)
  2229. {
  2230. enum zone_type high_zoneidx = gfp_zone(gfp_mask);
  2231. struct zone *preferred_zone;
  2232. struct page *page = NULL;
  2233. int migratetype = allocflags_to_migratetype(gfp_mask);
  2234. unsigned int cpuset_mems_cookie;
  2235. int alloc_flags = ALLOC_WMARK_LOW|ALLOC_CPUSET;
  2236. struct mem_cgroup *memcg = NULL;
  2237. gfp_mask &= gfp_allowed_mask;
  2238. lockdep_trace_alloc(gfp_mask);
  2239. might_sleep_if(gfp_mask & __GFP_WAIT);
  2240. if (should_fail_alloc_page(gfp_mask, order))
  2241. return NULL;
  2242. /*
  2243. * Check the zones suitable for the gfp_mask contain at least one
  2244. * valid zone. It's possible to have an empty zonelist as a result
  2245. * of GFP_THISNODE and a memoryless node
  2246. */
  2247. if (unlikely(!zonelist->_zonerefs->zone))
  2248. return NULL;
  2249. /*
  2250. * Will only have any effect when __GFP_KMEMCG is set. This is
  2251. * verified in the (always inline) callee
  2252. */
  2253. if (!memcg_kmem_newpage_charge(gfp_mask, &memcg, order))
  2254. return NULL;
  2255. retry_cpuset:
  2256. cpuset_mems_cookie = get_mems_allowed();
  2257. /* The preferred zone is used for statistics later */
  2258. first_zones_zonelist(zonelist, high_zoneidx,
  2259. nodemask ? : &cpuset_current_mems_allowed,
  2260. &preferred_zone);
  2261. if (!preferred_zone)
  2262. goto out;
  2263. #ifdef CONFIG_CMA
  2264. if (allocflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
  2265. alloc_flags |= ALLOC_CMA;
  2266. #endif
  2267. /* First allocation attempt */
  2268. page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask, order,
  2269. zonelist, high_zoneidx, alloc_flags,
  2270. preferred_zone, migratetype);
  2271. if (unlikely(!page))
  2272. page = __alloc_pages_slowpath(gfp_mask, order,
  2273. zonelist, high_zoneidx, nodemask,
  2274. preferred_zone, migratetype);
  2275. trace_mm_page_alloc(page, order, gfp_mask, migratetype);
  2276. out:
  2277. /*
  2278. * When updating a task's mems_allowed, it is possible to race with
  2279. * parallel threads in such a way that an allocation can fail while
  2280. * the mask is being updated. If a page allocation is about to fail,
  2281. * check if the cpuset changed during allocation and if so, retry.
  2282. */
  2283. if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
  2284. goto retry_cpuset;
  2285. memcg_kmem_commit_charge(page, memcg, order);
  2286. return page;
  2287. }
  2288. EXPORT_SYMBOL(__alloc_pages_nodemask);
  2289. /*
  2290. * Common helper functions.
  2291. */
  2292. unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
  2293. {
  2294. struct page *page;
  2295. /*
  2296. * __get_free_pages() returns a 32-bit address, which cannot represent
  2297. * a highmem page
  2298. */
  2299. VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0);
  2300. page = alloc_pages(gfp_mask, order);
  2301. if (!page)
  2302. return 0;
  2303. return (unsigned long) page_address(page);
  2304. }
  2305. EXPORT_SYMBOL(__get_free_pages);
  2306. unsigned long get_zeroed_page(gfp_t gfp_mask)
  2307. {
  2308. return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
  2309. }
  2310. EXPORT_SYMBOL(get_zeroed_page);
  2311. void __free_pages(struct page *page, unsigned int order)
  2312. {
  2313. if (put_page_testzero(page)) {
  2314. if (order == 0)
  2315. free_hot_cold_page(page, 0);
  2316. else
  2317. __free_pages_ok(page, order);
  2318. }
  2319. }
  2320. EXPORT_SYMBOL(__free_pages);
  2321. void free_pages(unsigned long addr, unsigned int order)
  2322. {
  2323. if (addr != 0) {
  2324. VM_BUG_ON(!virt_addr_valid((void *)addr));
  2325. __free_pages(virt_to_page((void *)addr), order);
  2326. }
  2327. }
  2328. EXPORT_SYMBOL(free_pages);
  2329. /*
  2330. * __free_memcg_kmem_pages and free_memcg_kmem_pages will free
  2331. * pages allocated with __GFP_KMEMCG.
  2332. *
  2333. * Those pages are accounted to a particular memcg, embedded in the
  2334. * corresponding page_cgroup. To avoid adding a hit in the allocator to search
  2335. * for that information only to find out that it is NULL for users who have no
  2336. * interest in that whatsoever, we provide these functions.
  2337. *
  2338. * The caller knows better which flags it relies on.
  2339. */
  2340. void __free_memcg_kmem_pages(struct page *page, unsigned int order)
  2341. {
  2342. memcg_kmem_uncharge_pages(page, order);
  2343. __free_pages(page, order);
  2344. }
  2345. void free_memcg_kmem_pages(unsigned long addr, unsigned int order)
  2346. {
  2347. if (addr != 0) {
  2348. VM_BUG_ON(!virt_addr_valid((void *)addr));
  2349. __free_memcg_kmem_pages(virt_to_page((void *)addr), order);
  2350. }
  2351. }
  2352. static void *make_alloc_exact(unsigned long addr, unsigned order, size_t size)
  2353. {
  2354. if (addr) {
  2355. unsigned long alloc_end = addr + (PAGE_SIZE << order);
  2356. unsigned long used = addr + PAGE_ALIGN(size);
  2357. split_page(virt_to_page((void *)addr), order);
  2358. while (used < alloc_end) {
  2359. free_page(used);
  2360. used += PAGE_SIZE;
  2361. }
  2362. }
  2363. return (void *)addr;
  2364. }
  2365. /**
  2366. * alloc_pages_exact - allocate an exact number physically-contiguous pages.
  2367. * @size: the number of bytes to allocate
  2368. * @gfp_mask: GFP flags for the allocation
  2369. *
  2370. * This function is similar to alloc_pages(), except that it allocates the
  2371. * minimum number of pages to satisfy the request. alloc_pages() can only
  2372. * allocate memory in power-of-two pages.
  2373. *
  2374. * This function is also limited by MAX_ORDER.
  2375. *
  2376. * Memory allocated by this function must be released by free_pages_exact().
  2377. */
  2378. void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
  2379. {
  2380. unsigned int order = get_order(size);
  2381. unsigned long addr;
  2382. addr = __get_free_pages(gfp_mask, order);
  2383. return make_alloc_exact(addr, order, size);
  2384. }
  2385. EXPORT_SYMBOL(alloc_pages_exact);
  2386. /**
  2387. * alloc_pages_exact_nid - allocate an exact number of physically-contiguous
  2388. * pages on a node.
  2389. * @nid: the preferred node ID where memory should be allocated
  2390. * @size: the number of bytes to allocate
  2391. * @gfp_mask: GFP flags for the allocation
  2392. *
  2393. * Like alloc_pages_exact(), but try to allocate on node nid first before falling
  2394. * back.
  2395. * Note this is not alloc_pages_exact_node() which allocates on a specific node,
  2396. * but is not exact.
  2397. */
  2398. void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask)
  2399. {
  2400. unsigned order = get_order(size);
  2401. struct page *p = alloc_pages_node(nid, gfp_mask, order);
  2402. if (!p)
  2403. return NULL;
  2404. return make_alloc_exact((unsigned long)page_address(p), order, size);
  2405. }
  2406. EXPORT_SYMBOL(alloc_pages_exact_nid);
  2407. /**
  2408. * free_pages_exact - release memory allocated via alloc_pages_exact()
  2409. * @virt: the value returned by alloc_pages_exact.
  2410. * @size: size of allocation, same value as passed to alloc_pages_exact().
  2411. *
  2412. * Release the memory allocated by a previous call to alloc_pages_exact.
  2413. */
  2414. void free_pages_exact(void *virt, size_t size)
  2415. {
  2416. unsigned long addr = (unsigned long)virt;
  2417. unsigned long end = addr + PAGE_ALIGN(size);
  2418. while (addr < end) {
  2419. free_page(addr);
  2420. addr += PAGE_SIZE;
  2421. }
  2422. }
  2423. EXPORT_SYMBOL(free_pages_exact);
  2424. static unsigned int nr_free_zone_pages(int offset)
  2425. {
  2426. struct zoneref *z;
  2427. struct zone *zone;
  2428. /* Just pick one node, since fallback list is circular */
  2429. unsigned int sum = 0;
  2430. struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
  2431. for_each_zone_zonelist(zone, z, zonelist, offset) {
  2432. unsigned long size = zone->present_pages;
  2433. unsigned long high = high_wmark_pages(zone);
  2434. if (size > high)
  2435. sum += size - high;
  2436. }
  2437. return sum;
  2438. }
  2439. /*
  2440. * Amount of free RAM allocatable within ZONE_DMA and ZONE_NORMAL
  2441. */
  2442. unsigned int nr_free_buffer_pages(void)
  2443. {
  2444. return nr_free_zone_pages(gfp_zone(GFP_USER));
  2445. }
  2446. EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
  2447. /*
  2448. * Amount of free RAM allocatable within all zones
  2449. */
  2450. unsigned int nr_free_pagecache_pages(void)
  2451. {
  2452. return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
  2453. }
  2454. static inline void show_node(struct zone *zone)
  2455. {
  2456. if (IS_ENABLED(CONFIG_NUMA))
  2457. printk("Node %d ", zone_to_nid(zone));
  2458. }
  2459. void si_meminfo(struct sysinfo *val)
  2460. {
  2461. val->totalram = totalram_pages;
  2462. val->sharedram = 0;
  2463. val->freeram = global_page_state(NR_FREE_PAGES);
  2464. val->bufferram = nr_blockdev_pages();
  2465. val->totalhigh = totalhigh_pages;
  2466. val->freehigh = nr_free_highpages();
  2467. val->mem_unit = PAGE_SIZE;
  2468. }
  2469. EXPORT_SYMBOL(si_meminfo);
  2470. #ifdef CONFIG_NUMA
  2471. void si_meminfo_node(struct sysinfo *val, int nid)
  2472. {
  2473. pg_data_t *pgdat = NODE_DATA(nid);
  2474. val->totalram = pgdat->node_present_pages;
  2475. val->freeram = node_page_state(nid, NR_FREE_PAGES);
  2476. #ifdef CONFIG_HIGHMEM
  2477. val->totalhigh = pgdat->node_zones[ZONE_HIGHMEM].present_pages;
  2478. val->freehigh = zone_page_state(&pgdat->node_zones[ZONE_HIGHMEM],
  2479. NR_FREE_PAGES);
  2480. #else
  2481. val->totalhigh = 0;
  2482. val->freehigh = 0;
  2483. #endif
  2484. val->mem_unit = PAGE_SIZE;
  2485. }
  2486. #endif
  2487. /*
  2488. * Determine whether the node should be displayed or not, depending on whether
  2489. * SHOW_MEM_FILTER_NODES was passed to show_free_areas().
  2490. */
  2491. bool skip_free_areas_node(unsigned int flags, int nid)
  2492. {
  2493. bool ret = false;
  2494. unsigned int cpuset_mems_cookie;
  2495. if (!(flags & SHOW_MEM_FILTER_NODES))
  2496. goto out;
  2497. do {
  2498. cpuset_mems_cookie = get_mems_allowed();
  2499. ret = !node_isset(nid, cpuset_current_mems_allowed);
  2500. } while (!put_mems_allowed(cpuset_mems_cookie));
  2501. out:
  2502. return ret;
  2503. }
  2504. #define K(x) ((x) << (PAGE_SHIFT-10))
  2505. static void show_migration_types(unsigned char type)
  2506. {
  2507. static const char types[MIGRATE_TYPES] = {
  2508. [MIGRATE_UNMOVABLE] = 'U',
  2509. [MIGRATE_RECLAIMABLE] = 'E',
  2510. [MIGRATE_MOVABLE] = 'M',
  2511. [MIGRATE_RESERVE] = 'R',
  2512. #ifdef CONFIG_CMA
  2513. [MIGRATE_CMA] = 'C',
  2514. #endif
  2515. [MIGRATE_ISOLATE] = 'I',
  2516. };
  2517. char tmp[MIGRATE_TYPES + 1];
  2518. char *p = tmp;
  2519. int i;
  2520. for (i = 0; i < MIGRATE_TYPES; i++) {
  2521. if (type & (1 << i))
  2522. *p++ = types[i];
  2523. }
  2524. *p = '\0';
  2525. printk("(%s) ", tmp);
  2526. }
  2527. /*
  2528. * Show free area list (used inside shift_scroll-lock stuff)
  2529. * We also calculate the percentage fragmentation. We do this by counting the
  2530. * memory on each free list with the exception of the first item on the list.
  2531. * Suppresses nodes that are not allowed by current's cpuset if
  2532. * SHOW_MEM_FILTER_NODES is passed.
  2533. */
  2534. void show_free_areas(unsigned int filter)
  2535. {
  2536. int cpu;
  2537. struct zone *zone;
  2538. for_each_populated_zone(zone) {
  2539. if (skip_free_areas_node(filter, zone_to_nid(zone)))
  2540. continue;
  2541. show_node(zone);
  2542. printk("%s per-cpu:\n", zone->name);
  2543. for_each_online_cpu(cpu) {
  2544. struct per_cpu_pageset *pageset;
  2545. pageset = per_cpu_ptr(zone->pageset, cpu);
  2546. printk("CPU %4d: hi:%5d, btch:%4d usd:%4d\n",
  2547. cpu, pageset->pcp.high,
  2548. pageset->pcp.batch, pageset->pcp.count);
  2549. }
  2550. }
  2551. printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n"
  2552. " active_file:%lu inactive_file:%lu isolated_file:%lu\n"
  2553. " unevictable:%lu"
  2554. " dirty:%lu writeback:%lu unstable:%lu\n"
  2555. " free:%lu slab_reclaimable:%lu slab_unreclaimable:%lu\n"
  2556. " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n"
  2557. " free_cma:%lu\n",
  2558. global_page_state(NR_ACTIVE_ANON),
  2559. global_page_state(NR_INACTIVE_ANON),
  2560. global_page_state(NR_ISOLATED_ANON),
  2561. global_page_state(NR_ACTIVE_FILE),
  2562. global_page_state(NR_INACTIVE_FILE),
  2563. global_page_state(NR_ISOLATED_FILE),
  2564. global_page_state(NR_UNEVICTABLE),
  2565. global_page_state(NR_FILE_DIRTY),
  2566. global_page_state(NR_WRITEBACK),
  2567. global_page_state(NR_UNSTABLE_NFS),
  2568. global_page_state(NR_FREE_PAGES),
  2569. global_page_state(NR_SLAB_RECLAIMABLE),
  2570. global_page_state(NR_SLAB_UNRECLAIMABLE),
  2571. global_page_state(NR_FILE_MAPPED),
  2572. global_page_state(NR_SHMEM),
  2573. global_page_state(NR_PAGETABLE),
  2574. global_page_state(NR_BOUNCE),
  2575. global_page_state(NR_FREE_CMA_PAGES));
  2576. for_each_populated_zone(zone) {
  2577. int i;
  2578. if (skip_free_areas_node(filter, zone_to_nid(zone)))
  2579. continue;
  2580. show_node(zone);
  2581. printk("%s"
  2582. " free:%lukB"
  2583. " min:%lukB"
  2584. " low:%lukB"
  2585. " high:%lukB"
  2586. " active_anon:%lukB"
  2587. " inactive_anon:%lukB"
  2588. " active_file:%lukB"
  2589. " inactive_file:%lukB"
  2590. " unevictable:%lukB"
  2591. " isolated(anon):%lukB"
  2592. " isolated(file):%lukB"
  2593. " present:%lukB"
  2594. " managed:%lukB"
  2595. " mlocked:%lukB"
  2596. " dirty:%lukB"
  2597. " writeback:%lukB"
  2598. " mapped:%lukB"
  2599. " shmem:%lukB"
  2600. " slab_reclaimable:%lukB"
  2601. " slab_unreclaimable:%lukB"
  2602. " kernel_stack:%lukB"
  2603. " pagetables:%lukB"
  2604. " unstable:%lukB"
  2605. " bounce:%lukB"
  2606. " free_cma:%lukB"
  2607. " writeback_tmp:%lukB"
  2608. " pages_scanned:%lu"
  2609. " all_unreclaimable? %s"
  2610. "\n",
  2611. zone->name,
  2612. K(zone_page_state(zone, NR_FREE_PAGES)),
  2613. K(min_wmark_pages(zone)),
  2614. K(low_wmark_pages(zone)),
  2615. K(high_wmark_pages(zone)),
  2616. K(zone_page_state(zone, NR_ACTIVE_ANON)),
  2617. K(zone_page_state(zone, NR_INACTIVE_ANON)),
  2618. K(zone_page_state(zone, NR_ACTIVE_FILE)),
  2619. K(zone_page_state(zone, NR_INACTIVE_FILE)),
  2620. K(zone_page_state(zone, NR_UNEVICTABLE)),
  2621. K(zone_page_state(zone, NR_ISOLATED_ANON)),
  2622. K(zone_page_state(zone, NR_ISOLATED_FILE)),
  2623. K(zone->present_pages),
  2624. K(zone->managed_pages),
  2625. K(zone_page_state(zone, NR_MLOCK)),
  2626. K(zone_page_state(zone, NR_FILE_DIRTY)),
  2627. K(zone_page_state(zone, NR_WRITEBACK)),
  2628. K(zone_page_state(zone, NR_FILE_MAPPED)),
  2629. K(zone_page_state(zone, NR_SHMEM)),
  2630. K(zone_page_state(zone, NR_SLAB_RECLAIMABLE)),
  2631. K(zone_page_state(zone, NR_SLAB_UNRECLAIMABLE)),
  2632. zone_page_state(zone, NR_KERNEL_STACK) *
  2633. THREAD_SIZE / 1024,
  2634. K(zone_page_state(zone, NR_PAGETABLE)),
  2635. K(zone_page_state(zone, NR_UNSTABLE_NFS)),
  2636. K(zone_page_state(zone, NR_BOUNCE)),
  2637. K(zone_page_state(zone, NR_FREE_CMA_PAGES)),
  2638. K(zone_page_state(zone, NR_WRITEBACK_TEMP)),
  2639. zone->pages_scanned,
  2640. (zone->all_unreclaimable ? "yes" : "no")
  2641. );
  2642. printk("lowmem_reserve[]:");
  2643. for (i = 0; i < MAX_NR_ZONES; i++)
  2644. printk(" %lu", zone->lowmem_reserve[i]);
  2645. printk("\n");
  2646. }
  2647. for_each_populated_zone(zone) {
  2648. unsigned long nr[MAX_ORDER], flags, order, total = 0;
  2649. unsigned char types[MAX_ORDER];
  2650. if (skip_free_areas_node(filter, zone_to_nid(zone)))
  2651. continue;
  2652. show_node(zone);
  2653. printk("%s: ", zone->name);
  2654. spin_lock_irqsave(&zone->lock, flags);
  2655. for (order = 0; order < MAX_ORDER; order++) {
  2656. struct free_area *area = &zone->free_area[order];
  2657. int type;
  2658. nr[order] = area->nr_free;
  2659. total += nr[order] << order;
  2660. types[order] = 0;
  2661. for (type = 0; type < MIGRATE_TYPES; type++) {
  2662. if (!list_empty(&area->free_list[type]))
  2663. types[order] |= 1 << type;
  2664. }
  2665. }
  2666. spin_unlock_irqrestore(&zone->lock, flags);
  2667. for (order = 0; order < MAX_ORDER; order++) {
  2668. printk("%lu*%lukB ", nr[order], K(1UL) << order);
  2669. if (nr[order])
  2670. show_migration_types(types[order]);
  2671. }
  2672. printk("= %lukB\n", K(total));
  2673. }
  2674. printk("%ld total pagecache pages\n", global_page_state(NR_FILE_PAGES));
  2675. show_swap_cache_info();
  2676. }
  2677. static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
  2678. {
  2679. zoneref->zone = zone;
  2680. zoneref->zone_idx = zone_idx(zone);
  2681. }
  2682. /*
  2683. * Builds allocation fallback zone lists.
  2684. *
  2685. * Add all populated zones of a node to the zonelist.
  2686. */
  2687. static int build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist,
  2688. int nr_zones, enum zone_type zone_type)
  2689. {
  2690. struct zone *zone;
  2691. BUG_ON(zone_type >= MAX_NR_ZONES);
  2692. zone_type++;
  2693. do {
  2694. zone_type--;
  2695. zone = pgdat->node_zones + zone_type;
  2696. if (populated_zone(zone)) {
  2697. zoneref_set_zone(zone,
  2698. &zonelist->_zonerefs[nr_zones++]);
  2699. check_highest_zone(zone_type);
  2700. }
  2701. } while (zone_type);
  2702. return nr_zones;
  2703. }
  2704. /*
  2705. * zonelist_order:
  2706. * 0 = automatic detection of better ordering.
  2707. * 1 = order by ([node] distance, -zonetype)
  2708. * 2 = order by (-zonetype, [node] distance)
  2709. *
  2710. * If not NUMA, ZONELIST_ORDER_ZONE and ZONELIST_ORDER_NODE will create
  2711. * the same zonelist. So only NUMA can configure this param.
  2712. */
  2713. #define ZONELIST_ORDER_DEFAULT 0
  2714. #define ZONELIST_ORDER_NODE 1
  2715. #define ZONELIST_ORDER_ZONE 2
  2716. /* zonelist order in the kernel.
  2717. * set_zonelist_order() will set this to NODE or ZONE.
  2718. */
  2719. static int current_zonelist_order = ZONELIST_ORDER_DEFAULT;
  2720. static char zonelist_order_name[3][8] = {"Default", "Node", "Zone"};
  2721. #ifdef CONFIG_NUMA
  2722. /* The value user specified ....changed by config */
  2723. static int user_zonelist_order = ZONELIST_ORDER_DEFAULT;
  2724. /* string for sysctl */
  2725. #define NUMA_ZONELIST_ORDER_LEN 16
  2726. char numa_zonelist_order[16] = "default";
  2727. /*
  2728. * interface for configure zonelist ordering.
  2729. * command line option "numa_zonelist_order"
  2730. * = "[dD]efault - default, automatic configuration.
  2731. * = "[nN]ode - order by node locality, then by zone within node
  2732. * = "[zZ]one - order by zone, then by locality within zone
  2733. */
  2734. static int __parse_numa_zonelist_order(char *s)
  2735. {
  2736. if (*s == 'd' || *s == 'D') {
  2737. user_zonelist_order = ZONELIST_ORDER_DEFAULT;
  2738. } else if (*s == 'n' || *s == 'N') {
  2739. user_zonelist_order = ZONELIST_ORDER_NODE;
  2740. } else if (*s == 'z' || *s == 'Z') {
  2741. user_zonelist_order = ZONELIST_ORDER_ZONE;
  2742. } else {
  2743. printk(KERN_WARNING
  2744. "Ignoring invalid numa_zonelist_order value: "
  2745. "%s\n", s);
  2746. return -EINVAL;
  2747. }
  2748. return 0;
  2749. }
  2750. static __init int setup_numa_zonelist_order(char *s)
  2751. {
  2752. int ret;
  2753. if (!s)
  2754. return 0;
  2755. ret = __parse_numa_zonelist_order(s);
  2756. if (ret == 0)
  2757. strlcpy(numa_zonelist_order, s, NUMA_ZONELIST_ORDER_LEN);
  2758. return ret;
  2759. }
  2760. early_param("numa_zonelist_order", setup_numa_zonelist_order);
  2761. /*
  2762. * sysctl handler for numa_zonelist_order
  2763. */
  2764. int numa_zonelist_order_handler(ctl_table *table, int write,
  2765. void __user *buffer, size_t *length,
  2766. loff_t *ppos)
  2767. {
  2768. char saved_string[NUMA_ZONELIST_ORDER_LEN];
  2769. int ret;
  2770. static DEFINE_MUTEX(zl_order_mutex);
  2771. mutex_lock(&zl_order_mutex);
  2772. if (write)
  2773. strcpy(saved_string, (char*)table->data);
  2774. ret = proc_dostring(table, write, buffer, length, ppos);
  2775. if (ret)
  2776. goto out;
  2777. if (write) {
  2778. int oldval = user_zonelist_order;
  2779. if (__parse_numa_zonelist_order((char*)table->data)) {
  2780. /*
  2781. * bogus value. restore saved string
  2782. */
  2783. strncpy((char*)table->data, saved_string,
  2784. NUMA_ZONELIST_ORDER_LEN);
  2785. user_zonelist_order = oldval;
  2786. } else if (oldval != user_zonelist_order) {
  2787. mutex_lock(&zonelists_mutex);
  2788. build_all_zonelists(NULL, NULL);
  2789. mutex_unlock(&zonelists_mutex);
  2790. }
  2791. }
  2792. out:
  2793. mutex_unlock(&zl_order_mutex);
  2794. return ret;
  2795. }
  2796. #define MAX_NODE_LOAD (nr_online_nodes)
  2797. static int node_load[MAX_NUMNODES];
  2798. /**
  2799. * find_next_best_node - find the next node that should appear in a given node's fallback list
  2800. * @node: node whose fallback list we're appending
  2801. * @used_node_mask: nodemask_t of already used nodes
  2802. *
  2803. * We use a number of factors to determine which is the next node that should
  2804. * appear on a given node's fallback list. The node should not have appeared
  2805. * already in @node's fallback list, and it should be the next closest node
  2806. * according to the distance array (which contains arbitrary distance values
  2807. * from each node to each node in the system), and should also prefer nodes
  2808. * with no CPUs, since presumably they'll have very little allocation pressure
  2809. * on them otherwise.
  2810. * It returns -1 if no node is found.
  2811. */
  2812. static int find_next_best_node(int node, nodemask_t *used_node_mask)
  2813. {
  2814. int n, val;
  2815. int min_val = INT_MAX;
  2816. int best_node = -1;
  2817. const struct cpumask *tmp = cpumask_of_node(0);
  2818. /* Use the local node if we haven't already */
  2819. if (!node_isset(node, *used_node_mask)) {
  2820. node_set(node, *used_node_mask);
  2821. return node;
  2822. }
  2823. for_each_node_state(n, N_MEMORY) {
  2824. /* Don't want a node to appear more than once */
  2825. if (node_isset(n, *used_node_mask))
  2826. continue;
  2827. /* Use the distance array to find the distance */
  2828. val = node_distance(node, n);
  2829. /* Penalize nodes under us ("prefer the next node") */
  2830. val += (n < node);
  2831. /* Give preference to headless and unused nodes */
  2832. tmp = cpumask_of_node(n);
  2833. if (!cpumask_empty(tmp))
  2834. val += PENALTY_FOR_NODE_WITH_CPUS;
  2835. /* Slight preference for less loaded node */
  2836. val *= (MAX_NODE_LOAD*MAX_NUMNODES);
  2837. val += node_load[n];
  2838. if (val < min_val) {
  2839. min_val = val;
  2840. best_node = n;
  2841. }
  2842. }
  2843. if (best_node >= 0)
  2844. node_set(best_node, *used_node_mask);
  2845. return best_node;
  2846. }
  2847. /*
  2848. * Build zonelists ordered by node and zones within node.
  2849. * This results in maximum locality--normal zone overflows into local
  2850. * DMA zone, if any--but risks exhausting DMA zone.
  2851. */
  2852. static void build_zonelists_in_node_order(pg_data_t *pgdat, int node)
  2853. {
  2854. int j;
  2855. struct zonelist *zonelist;
  2856. zonelist = &pgdat->node_zonelists[0];
  2857. for (j = 0; zonelist->_zonerefs[j].zone != NULL; j++)
  2858. ;
  2859. j = build_zonelists_node(NODE_DATA(node), zonelist, j,
  2860. MAX_NR_ZONES - 1);
  2861. zonelist->_zonerefs[j].zone = NULL;
  2862. zonelist->_zonerefs[j].zone_idx = 0;
  2863. }
  2864. /*
  2865. * Build gfp_thisnode zonelists
  2866. */
  2867. static void build_thisnode_zonelists(pg_data_t *pgdat)
  2868. {
  2869. int j;
  2870. struct zonelist *zonelist;
  2871. zonelist = &pgdat->node_zonelists[1];
  2872. j = build_zonelists_node(pgdat, zonelist, 0, MAX_NR_ZONES - 1);
  2873. zonelist->_zonerefs[j].zone = NULL;
  2874. zonelist->_zonerefs[j].zone_idx = 0;
  2875. }
  2876. /*
  2877. * Build zonelists ordered by zone and nodes within zones.
  2878. * This results in conserving DMA zone[s] until all Normal memory is
  2879. * exhausted, but results in overflowing to remote node while memory
  2880. * may still exist in local DMA zone.
  2881. */
  2882. static int node_order[MAX_NUMNODES];
  2883. static void build_zonelists_in_zone_order(pg_data_t *pgdat, int nr_nodes)
  2884. {
  2885. int pos, j, node;
  2886. int zone_type; /* needs to be signed */
  2887. struct zone *z;
  2888. struct zonelist *zonelist;
  2889. zonelist = &pgdat->node_zonelists[0];
  2890. pos = 0;
  2891. for (zone_type = MAX_NR_ZONES - 1; zone_type >= 0; zone_type--) {
  2892. for (j = 0; j < nr_nodes; j++) {
  2893. node = node_order[j];
  2894. z = &NODE_DATA(node)->node_zones[zone_type];
  2895. if (populated_zone(z)) {
  2896. zoneref_set_zone(z,
  2897. &zonelist->_zonerefs[pos++]);
  2898. check_highest_zone(zone_type);
  2899. }
  2900. }
  2901. }
  2902. zonelist->_zonerefs[pos].zone = NULL;
  2903. zonelist->_zonerefs[pos].zone_idx = 0;
  2904. }
  2905. static int default_zonelist_order(void)
  2906. {
  2907. int nid, zone_type;
  2908. unsigned long low_kmem_size,total_size;
  2909. struct zone *z;
  2910. int average_size;
  2911. /*
  2912. * ZONE_DMA and ZONE_DMA32 can be very small area in the system.
  2913. * If they are really small and used heavily, the system can fall
  2914. * into OOM very easily.
  2915. * This function detect ZONE_DMA/DMA32 size and configures zone order.
  2916. */
  2917. /* Is there ZONE_NORMAL ? (ex. ppc has only DMA zone..) */
  2918. low_kmem_size = 0;
  2919. total_size = 0;
  2920. for_each_online_node(nid) {
  2921. for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
  2922. z = &NODE_DATA(nid)->node_zones[zone_type];
  2923. if (populated_zone(z)) {
  2924. if (zone_type < ZONE_NORMAL)
  2925. low_kmem_size += z->present_pages;
  2926. total_size += z->present_pages;
  2927. } else if (zone_type == ZONE_NORMAL) {
  2928. /*
  2929. * If any node has only lowmem, then node order
  2930. * is preferred to allow kernel allocations
  2931. * locally; otherwise, they can easily infringe
  2932. * on other nodes when there is an abundance of
  2933. * lowmem available to allocate from.
  2934. */
  2935. return ZONELIST_ORDER_NODE;
  2936. }
  2937. }
  2938. }
  2939. if (!low_kmem_size || /* there are no DMA area. */
  2940. low_kmem_size > total_size/2) /* DMA/DMA32 is big. */
  2941. return ZONELIST_ORDER_NODE;
  2942. /*
  2943. * look into each node's config.
  2944. * If there is a node whose DMA/DMA32 memory is very big area on
  2945. * local memory, NODE_ORDER may be suitable.
  2946. */
  2947. average_size = total_size /
  2948. (nodes_weight(node_states[N_MEMORY]) + 1);
  2949. for_each_online_node(nid) {
  2950. low_kmem_size = 0;
  2951. total_size = 0;
  2952. for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
  2953. z = &NODE_DATA(nid)->node_zones[zone_type];
  2954. if (populated_zone(z)) {
  2955. if (zone_type < ZONE_NORMAL)
  2956. low_kmem_size += z->present_pages;
  2957. total_size += z->present_pages;
  2958. }
  2959. }
  2960. if (low_kmem_size &&
  2961. total_size > average_size && /* ignore small node */
  2962. low_kmem_size > total_size * 70/100)
  2963. return ZONELIST_ORDER_NODE;
  2964. }
  2965. return ZONELIST_ORDER_ZONE;
  2966. }
  2967. static void set_zonelist_order(void)
  2968. {
  2969. if (user_zonelist_order == ZONELIST_ORDER_DEFAULT)
  2970. current_zonelist_order = default_zonelist_order();
  2971. else
  2972. current_zonelist_order = user_zonelist_order;
  2973. }
  2974. static void build_zonelists(pg_data_t *pgdat)
  2975. {
  2976. int j, node, load;
  2977. enum zone_type i;
  2978. nodemask_t used_mask;
  2979. int local_node, prev_node;
  2980. struct zonelist *zonelist;
  2981. int order = current_zonelist_order;
  2982. /* initialize zonelists */
  2983. for (i = 0; i < MAX_ZONELISTS; i++) {
  2984. zonelist = pgdat->node_zonelists + i;
  2985. zonelist->_zonerefs[0].zone = NULL;
  2986. zonelist->_zonerefs[0].zone_idx = 0;
  2987. }
  2988. /* NUMA-aware ordering of nodes */
  2989. local_node = pgdat->node_id;
  2990. load = nr_online_nodes;
  2991. prev_node = local_node;
  2992. nodes_clear(used_mask);
  2993. memset(node_order, 0, sizeof(node_order));
  2994. j = 0;
  2995. while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
  2996. /*
  2997. * We don't want to pressure a particular node.
  2998. * So adding penalty to the first node in same
  2999. * distance group to make it round-robin.
  3000. */
  3001. if (node_distance(local_node, node) !=
  3002. node_distance(local_node, prev_node))
  3003. node_load[node] = load;
  3004. prev_node = node;
  3005. load--;
  3006. if (order == ZONELIST_ORDER_NODE)
  3007. build_zonelists_in_node_order(pgdat, node);
  3008. else
  3009. node_order[j++] = node; /* remember order */
  3010. }
  3011. if (order == ZONELIST_ORDER_ZONE) {
  3012. /* calculate node order -- i.e., DMA last! */
  3013. build_zonelists_in_zone_order(pgdat, j);
  3014. }
  3015. build_thisnode_zonelists(pgdat);
  3016. }
  3017. /* Construct the zonelist performance cache - see further mmzone.h */
  3018. static void build_zonelist_cache(pg_data_t *pgdat)
  3019. {
  3020. struct zonelist *zonelist;
  3021. struct zonelist_cache *zlc;
  3022. struct zoneref *z;
  3023. zonelist = &pgdat->node_zonelists[0];
  3024. zonelist->zlcache_ptr = zlc = &zonelist->zlcache;
  3025. bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
  3026. for (z = zonelist->_zonerefs; z->zone; z++)
  3027. zlc->z_to_n[z - zonelist->_zonerefs] = zonelist_node_idx(z);
  3028. }
  3029. #ifdef CONFIG_HAVE_MEMORYLESS_NODES
  3030. /*
  3031. * Return node id of node used for "local" allocations.
  3032. * I.e., first node id of first zone in arg node's generic zonelist.
  3033. * Used for initializing percpu 'numa_mem', which is used primarily
  3034. * for kernel allocations, so use GFP_KERNEL flags to locate zonelist.
  3035. */
  3036. int local_memory_node(int node)
  3037. {
  3038. struct zone *zone;
  3039. (void)first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
  3040. gfp_zone(GFP_KERNEL),
  3041. NULL,
  3042. &zone);
  3043. return zone->node;
  3044. }
  3045. #endif
  3046. #else /* CONFIG_NUMA */
  3047. static void set_zonelist_order(void)
  3048. {
  3049. current_zonelist_order = ZONELIST_ORDER_ZONE;
  3050. }
  3051. static void build_zonelists(pg_data_t *pgdat)
  3052. {
  3053. int node, local_node;
  3054. enum zone_type j;
  3055. struct zonelist *zonelist;
  3056. local_node = pgdat->node_id;
  3057. zonelist = &pgdat->node_zonelists[0];
  3058. j = build_zonelists_node(pgdat, zonelist, 0, MAX_NR_ZONES - 1);
  3059. /*
  3060. * Now we build the zonelist so that it contains the zones
  3061. * of all the other nodes.
  3062. * We don't want to pressure a particular node, so when
  3063. * building the zones for node N, we make sure that the
  3064. * zones coming right after the local ones are those from
  3065. * node N+1 (modulo N)
  3066. */
  3067. for (node = local_node + 1; node < MAX_NUMNODES; node++) {
  3068. if (!node_online(node))
  3069. continue;
  3070. j = build_zonelists_node(NODE_DATA(node), zonelist, j,
  3071. MAX_NR_ZONES - 1);
  3072. }
  3073. for (node = 0; node < local_node; node++) {
  3074. if (!node_online(node))
  3075. continue;
  3076. j = build_zonelists_node(NODE_DATA(node), zonelist, j,
  3077. MAX_NR_ZONES - 1);
  3078. }
  3079. zonelist->_zonerefs[j].zone = NULL;
  3080. zonelist->_zonerefs[j].zone_idx = 0;
  3081. }
  3082. /* non-NUMA variant of zonelist performance cache - just NULL zlcache_ptr */
  3083. static void build_zonelist_cache(pg_data_t *pgdat)
  3084. {
  3085. pgdat->node_zonelists[0].zlcache_ptr = NULL;
  3086. }
  3087. #endif /* CONFIG_NUMA */
  3088. /*
  3089. * Boot pageset table. One per cpu which is going to be used for all
  3090. * zones and all nodes. The parameters will be set in such a way
  3091. * that an item put on a list will immediately be handed over to
  3092. * the buddy list. This is safe since pageset manipulation is done
  3093. * with interrupts disabled.
  3094. *
  3095. * The boot_pagesets must be kept even after bootup is complete for
  3096. * unused processors and/or zones. They do play a role for bootstrapping
  3097. * hotplugged processors.
  3098. *
  3099. * zoneinfo_show() and maybe other functions do
  3100. * not check if the processor is online before following the pageset pointer.
  3101. * Other parts of the kernel may not check if the zone is available.
  3102. */
  3103. static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
  3104. static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
  3105. static void setup_zone_pageset(struct zone *zone);
  3106. /*
  3107. * Global mutex to protect against size modification of zonelists
  3108. * as well as to serialize pageset setup for the new populated zone.
  3109. */
  3110. DEFINE_MUTEX(zonelists_mutex);
  3111. /* return values int ....just for stop_machine() */
  3112. static int __build_all_zonelists(void *data)
  3113. {
  3114. int nid;
  3115. int cpu;
  3116. pg_data_t *self = data;
  3117. #ifdef CONFIG_NUMA
  3118. memset(node_load, 0, sizeof(node_load));
  3119. #endif
  3120. if (self && !node_online(self->node_id)) {
  3121. build_zonelists(self);
  3122. build_zonelist_cache(self);
  3123. }
  3124. for_each_online_node(nid) {
  3125. pg_data_t *pgdat = NODE_DATA(nid);
  3126. build_zonelists(pgdat);
  3127. build_zonelist_cache(pgdat);
  3128. }
  3129. /*
  3130. * Initialize the boot_pagesets that are going to be used
  3131. * for bootstrapping processors. The real pagesets for
  3132. * each zone will be allocated later when the per cpu
  3133. * allocator is available.
  3134. *
  3135. * boot_pagesets are used also for bootstrapping offline
  3136. * cpus if the system is already booted because the pagesets
  3137. * are needed to initialize allocators on a specific cpu too.
  3138. * F.e. the percpu allocator needs the page allocator which
  3139. * needs the percpu allocator in order to allocate its pagesets
  3140. * (a chicken-egg dilemma).
  3141. */
  3142. for_each_possible_cpu(cpu) {
  3143. setup_pageset(&per_cpu(boot_pageset, cpu), 0);
  3144. #ifdef CONFIG_HAVE_MEMORYLESS_NODES
  3145. /*
  3146. * We now know the "local memory node" for each node--
  3147. * i.e., the node of the first zone in the generic zonelist.
  3148. * Set up numa_mem percpu variable for on-line cpus. During
  3149. * boot, only the boot cpu should be on-line; we'll init the
  3150. * secondary cpus' numa_mem as they come on-line. During
  3151. * node/memory hotplug, we'll fixup all on-line cpus.
  3152. */
  3153. if (cpu_online(cpu))
  3154. set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu)));
  3155. #endif
  3156. }
  3157. return 0;
  3158. }
  3159. /*
  3160. * Called with zonelists_mutex held always
  3161. * unless system_state == SYSTEM_BOOTING.
  3162. */
  3163. void __ref build_all_zonelists(pg_data_t *pgdat, struct zone *zone)
  3164. {
  3165. set_zonelist_order();
  3166. if (system_state == SYSTEM_BOOTING) {
  3167. __build_all_zonelists(NULL);
  3168. mminit_verify_zonelist();
  3169. cpuset_init_current_mems_allowed();
  3170. } else {
  3171. /* we have to stop all cpus to guarantee there is no user
  3172. of zonelist */
  3173. #ifdef CONFIG_MEMORY_HOTPLUG
  3174. if (zone)
  3175. setup_zone_pageset(zone);
  3176. #endif
  3177. stop_machine(__build_all_zonelists, pgdat, NULL);
  3178. /* cpuset refresh routine should be here */
  3179. }
  3180. vm_total_pages = nr_free_pagecache_pages();
  3181. /*
  3182. * Disable grouping by mobility if the number of pages in the
  3183. * system is too low to allow the mechanism to work. It would be
  3184. * more accurate, but expensive to check per-zone. This check is
  3185. * made on memory-hotadd so a system can start with mobility
  3186. * disabled and enable it later
  3187. */
  3188. if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
  3189. page_group_by_mobility_disabled = 1;
  3190. else
  3191. page_group_by_mobility_disabled = 0;
  3192. printk("Built %i zonelists in %s order, mobility grouping %s. "
  3193. "Total pages: %ld\n",
  3194. nr_online_nodes,
  3195. zonelist_order_name[current_zonelist_order],
  3196. page_group_by_mobility_disabled ? "off" : "on",
  3197. vm_total_pages);
  3198. #ifdef CONFIG_NUMA
  3199. printk("Policy zone: %s\n", zone_names[policy_zone]);
  3200. #endif
  3201. }
  3202. /*
  3203. * Helper functions to size the waitqueue hash table.
  3204. * Essentially these want to choose hash table sizes sufficiently
  3205. * large so that collisions trying to wait on pages are rare.
  3206. * But in fact, the number of active page waitqueues on typical
  3207. * systems is ridiculously low, less than 200. So this is even
  3208. * conservative, even though it seems large.
  3209. *
  3210. * The constant PAGES_PER_WAITQUEUE specifies the ratio of pages to
  3211. * waitqueues, i.e. the size of the waitq table given the number of pages.
  3212. */
  3213. #define PAGES_PER_WAITQUEUE 256
  3214. #ifndef CONFIG_MEMORY_HOTPLUG
  3215. static inline unsigned long wait_table_hash_nr_entries(unsigned long pages)
  3216. {
  3217. unsigned long size = 1;
  3218. pages /= PAGES_PER_WAITQUEUE;
  3219. while (size < pages)
  3220. size <<= 1;
  3221. /*
  3222. * Once we have dozens or even hundreds of threads sleeping
  3223. * on IO we've got bigger problems than wait queue collision.
  3224. * Limit the size of the wait table to a reasonable size.
  3225. */
  3226. size = min(size, 4096UL);
  3227. return max(size, 4UL);
  3228. }
  3229. #else
  3230. /*
  3231. * A zone's size might be changed by hot-add, so it is not possible to determine
  3232. * a suitable size for its wait_table. So we use the maximum size now.
  3233. *
  3234. * The max wait table size = 4096 x sizeof(wait_queue_head_t). ie:
  3235. *
  3236. * i386 (preemption config) : 4096 x 16 = 64Kbyte.
  3237. * ia64, x86-64 (no preemption): 4096 x 20 = 80Kbyte.
  3238. * ia64, x86-64 (preemption) : 4096 x 24 = 96Kbyte.
  3239. *
  3240. * The maximum entries are prepared when a zone's memory is (512K + 256) pages
  3241. * or more by the traditional way. (See above). It equals:
  3242. *
  3243. * i386, x86-64, powerpc(4K page size) : = ( 2G + 1M)byte.
  3244. * ia64(16K page size) : = ( 8G + 4M)byte.
  3245. * powerpc (64K page size) : = (32G +16M)byte.
  3246. */
  3247. static inline unsigned long wait_table_hash_nr_entries(unsigned long pages)
  3248. {
  3249. return 4096UL;
  3250. }
  3251. #endif
  3252. /*
  3253. * This is an integer logarithm so that shifts can be used later
  3254. * to extract the more random high bits from the multiplicative
  3255. * hash function before the remainder is taken.
  3256. */
  3257. static inline unsigned long wait_table_bits(unsigned long size)
  3258. {
  3259. return ffz(~size);
  3260. }
  3261. #define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1))
  3262. /*
  3263. * Check if a pageblock contains reserved pages
  3264. */
  3265. static int pageblock_is_reserved(unsigned long start_pfn, unsigned long end_pfn)
  3266. {
  3267. unsigned long pfn;
  3268. for (pfn = start_pfn; pfn < end_pfn; pfn++) {
  3269. if (!pfn_valid_within(pfn) || PageReserved(pfn_to_page(pfn)))
  3270. return 1;
  3271. }
  3272. return 0;
  3273. }
  3274. /*
  3275. * Mark a number of pageblocks as MIGRATE_RESERVE. The number
  3276. * of blocks reserved is based on min_wmark_pages(zone). The memory within
  3277. * the reserve will tend to store contiguous free pages. Setting min_free_kbytes
  3278. * higher will lead to a bigger reserve which will get freed as contiguous
  3279. * blocks as reclaim kicks in
  3280. */
  3281. static void setup_zone_migrate_reserve(struct zone *zone)
  3282. {
  3283. unsigned long start_pfn, pfn, end_pfn, block_end_pfn;
  3284. struct page *page;
  3285. unsigned long block_migratetype;
  3286. int reserve;
  3287. /*
  3288. * Get the start pfn, end pfn and the number of blocks to reserve
  3289. * We have to be careful to be aligned to pageblock_nr_pages to
  3290. * make sure that we always check pfn_valid for the first page in
  3291. * the block.
  3292. */
  3293. start_pfn = zone->zone_start_pfn;
  3294. end_pfn = start_pfn + zone->spanned_pages;
  3295. start_pfn = roundup(start_pfn, pageblock_nr_pages);
  3296. reserve = roundup(min_wmark_pages(zone), pageblock_nr_pages) >>
  3297. pageblock_order;
  3298. /*
  3299. * Reserve blocks are generally in place to help high-order atomic
  3300. * allocations that are short-lived. A min_free_kbytes value that
  3301. * would result in more than 2 reserve blocks for atomic allocations
  3302. * is assumed to be in place to help anti-fragmentation for the
  3303. * future allocation of hugepages at runtime.
  3304. */
  3305. reserve = min(2, reserve);
  3306. for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
  3307. if (!pfn_valid(pfn))
  3308. continue;
  3309. page = pfn_to_page(pfn);
  3310. /* Watch out for overlapping nodes */
  3311. if (page_to_nid(page) != zone_to_nid(zone))
  3312. continue;
  3313. block_migratetype = get_pageblock_migratetype(page);
  3314. /* Only test what is necessary when the reserves are not met */
  3315. if (reserve > 0) {
  3316. /*
  3317. * Blocks with reserved pages will never free, skip
  3318. * them.
  3319. */
  3320. block_end_pfn = min(pfn + pageblock_nr_pages, end_pfn);
  3321. if (pageblock_is_reserved(pfn, block_end_pfn))
  3322. continue;
  3323. /* If this block is reserved, account for it */
  3324. if (block_migratetype == MIGRATE_RESERVE) {
  3325. reserve--;
  3326. continue;
  3327. }
  3328. /* Suitable for reserving if this block is movable */
  3329. if (block_migratetype == MIGRATE_MOVABLE) {
  3330. set_pageblock_migratetype(page,
  3331. MIGRATE_RESERVE);
  3332. move_freepages_block(zone, page,
  3333. MIGRATE_RESERVE);
  3334. reserve--;
  3335. continue;
  3336. }
  3337. }
  3338. /*
  3339. * If the reserve is met and this is a previous reserved block,
  3340. * take it back
  3341. */
  3342. if (block_migratetype == MIGRATE_RESERVE) {
  3343. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  3344. move_freepages_block(zone, page, MIGRATE_MOVABLE);
  3345. }
  3346. }
  3347. }
  3348. /*
  3349. * Initially all pages are reserved - free ones are freed
  3350. * up by free_all_bootmem() once the early boot process is
  3351. * done. Non-atomic initialization, single-pass.
  3352. */
  3353. void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
  3354. unsigned long start_pfn, enum memmap_context context)
  3355. {
  3356. struct page *page;
  3357. unsigned long end_pfn = start_pfn + size;
  3358. unsigned long pfn;
  3359. struct zone *z;
  3360. if (highest_memmap_pfn < end_pfn - 1)
  3361. highest_memmap_pfn = end_pfn - 1;
  3362. z = &NODE_DATA(nid)->node_zones[zone];
  3363. for (pfn = start_pfn; pfn < end_pfn; pfn++) {
  3364. /*
  3365. * There can be holes in boot-time mem_map[]s
  3366. * handed to this function. They do not
  3367. * exist on hotplugged memory.
  3368. */
  3369. if (context == MEMMAP_EARLY) {
  3370. if (!early_pfn_valid(pfn))
  3371. continue;
  3372. if (!early_pfn_in_nid(pfn, nid))
  3373. continue;
  3374. }
  3375. page = pfn_to_page(pfn);
  3376. set_page_links(page, zone, nid, pfn);
  3377. mminit_verify_page_links(page, zone, nid, pfn);
  3378. init_page_count(page);
  3379. reset_page_mapcount(page);
  3380. reset_page_last_nid(page);
  3381. SetPageReserved(page);
  3382. /*
  3383. * Mark the block movable so that blocks are reserved for
  3384. * movable at startup. This will force kernel allocations
  3385. * to reserve their blocks rather than leaking throughout
  3386. * the address space during boot when many long-lived
  3387. * kernel allocations are made. Later some blocks near
  3388. * the start are marked MIGRATE_RESERVE by
  3389. * setup_zone_migrate_reserve()
  3390. *
  3391. * bitmap is created for zone's valid pfn range. but memmap
  3392. * can be created for invalid pages (for alignment)
  3393. * check here not to call set_pageblock_migratetype() against
  3394. * pfn out of zone.
  3395. */
  3396. if ((z->zone_start_pfn <= pfn)
  3397. && (pfn < z->zone_start_pfn + z->spanned_pages)
  3398. && !(pfn & (pageblock_nr_pages - 1)))
  3399. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  3400. INIT_LIST_HEAD(&page->lru);
  3401. #ifdef WANT_PAGE_VIRTUAL
  3402. /* The shift won't overflow because ZONE_NORMAL is below 4G. */
  3403. if (!is_highmem_idx(zone))
  3404. set_page_address(page, __va(pfn << PAGE_SHIFT));
  3405. #endif
  3406. }
  3407. }
  3408. static void __meminit zone_init_free_lists(struct zone *zone)
  3409. {
  3410. int order, t;
  3411. for_each_migratetype_order(order, t) {
  3412. INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
  3413. zone->free_area[order].nr_free = 0;
  3414. }
  3415. }
  3416. #ifndef __HAVE_ARCH_MEMMAP_INIT
  3417. #define memmap_init(size, nid, zone, start_pfn) \
  3418. memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY)
  3419. #endif
  3420. static int __meminit zone_batchsize(struct zone *zone)
  3421. {
  3422. #ifdef CONFIG_MMU
  3423. int batch;
  3424. /*
  3425. * The per-cpu-pages pools are set to around 1000th of the
  3426. * size of the zone. But no more than 1/2 of a meg.
  3427. *
  3428. * OK, so we don't know how big the cache is. So guess.
  3429. */
  3430. batch = zone->present_pages / 1024;
  3431. if (batch * PAGE_SIZE > 512 * 1024)
  3432. batch = (512 * 1024) / PAGE_SIZE;
  3433. batch /= 4; /* We effectively *= 4 below */
  3434. if (batch < 1)
  3435. batch = 1;
  3436. /*
  3437. * Clamp the batch to a 2^n - 1 value. Having a power
  3438. * of 2 value was found to be more likely to have
  3439. * suboptimal cache aliasing properties in some cases.
  3440. *
  3441. * For example if 2 tasks are alternately allocating
  3442. * batches of pages, one task can end up with a lot
  3443. * of pages of one half of the possible page colors
  3444. * and the other with pages of the other colors.
  3445. */
  3446. batch = rounddown_pow_of_two(batch + batch/2) - 1;
  3447. return batch;
  3448. #else
  3449. /* The deferral and batching of frees should be suppressed under NOMMU
  3450. * conditions.
  3451. *
  3452. * The problem is that NOMMU needs to be able to allocate large chunks
  3453. * of contiguous memory as there's no hardware page translation to
  3454. * assemble apparent contiguous memory from discontiguous pages.
  3455. *
  3456. * Queueing large contiguous runs of pages for batching, however,
  3457. * causes the pages to actually be freed in smaller chunks. As there
  3458. * can be a significant delay between the individual batches being
  3459. * recycled, this leads to the once large chunks of space being
  3460. * fragmented and becoming unavailable for high-order allocations.
  3461. */
  3462. return 0;
  3463. #endif
  3464. }
  3465. static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
  3466. {
  3467. struct per_cpu_pages *pcp;
  3468. int migratetype;
  3469. memset(p, 0, sizeof(*p));
  3470. pcp = &p->pcp;
  3471. pcp->count = 0;
  3472. pcp->high = 6 * batch;
  3473. pcp->batch = max(1UL, 1 * batch);
  3474. for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++)
  3475. INIT_LIST_HEAD(&pcp->lists[migratetype]);
  3476. }
  3477. /*
  3478. * setup_pagelist_highmark() sets the high water mark for hot per_cpu_pagelist
  3479. * to the value high for the pageset p.
  3480. */
  3481. static void setup_pagelist_highmark(struct per_cpu_pageset *p,
  3482. unsigned long high)
  3483. {
  3484. struct per_cpu_pages *pcp;
  3485. pcp = &p->pcp;
  3486. pcp->high = high;
  3487. pcp->batch = max(1UL, high/4);
  3488. if ((high/4) > (PAGE_SHIFT * 8))
  3489. pcp->batch = PAGE_SHIFT * 8;
  3490. }
  3491. static void __meminit setup_zone_pageset(struct zone *zone)
  3492. {
  3493. int cpu;
  3494. zone->pageset = alloc_percpu(struct per_cpu_pageset);
  3495. for_each_possible_cpu(cpu) {
  3496. struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);
  3497. setup_pageset(pcp, zone_batchsize(zone));
  3498. if (percpu_pagelist_fraction)
  3499. setup_pagelist_highmark(pcp,
  3500. (zone->present_pages /
  3501. percpu_pagelist_fraction));
  3502. }
  3503. }
  3504. /*
  3505. * Allocate per cpu pagesets and initialize them.
  3506. * Before this call only boot pagesets were available.
  3507. */
  3508. void __init setup_per_cpu_pageset(void)
  3509. {
  3510. struct zone *zone;
  3511. for_each_populated_zone(zone)
  3512. setup_zone_pageset(zone);
  3513. }
  3514. static noinline __init_refok
  3515. int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages)
  3516. {
  3517. int i;
  3518. struct pglist_data *pgdat = zone->zone_pgdat;
  3519. size_t alloc_size;
  3520. /*
  3521. * The per-page waitqueue mechanism uses hashed waitqueues
  3522. * per zone.
  3523. */
  3524. zone->wait_table_hash_nr_entries =
  3525. wait_table_hash_nr_entries(zone_size_pages);
  3526. zone->wait_table_bits =
  3527. wait_table_bits(zone->wait_table_hash_nr_entries);
  3528. alloc_size = zone->wait_table_hash_nr_entries
  3529. * sizeof(wait_queue_head_t);
  3530. if (!slab_is_available()) {
  3531. zone->wait_table = (wait_queue_head_t *)
  3532. alloc_bootmem_node_nopanic(pgdat, alloc_size);
  3533. } else {
  3534. /*
  3535. * This case means that a zone whose size was 0 gets new memory
  3536. * via memory hot-add.
  3537. * But it may be the case that a new node was hot-added. In
  3538. * this case vmalloc() will not be able to use this new node's
  3539. * memory - this wait_table must be initialized to use this new
  3540. * node itself as well.
  3541. * To use this new node's memory, further consideration will be
  3542. * necessary.
  3543. */
  3544. zone->wait_table = vmalloc(alloc_size);
  3545. }
  3546. if (!zone->wait_table)
  3547. return -ENOMEM;
  3548. for(i = 0; i < zone->wait_table_hash_nr_entries; ++i)
  3549. init_waitqueue_head(zone->wait_table + i);
  3550. return 0;
  3551. }
  3552. static __meminit void zone_pcp_init(struct zone *zone)
  3553. {
  3554. /*
  3555. * per cpu subsystem is not up at this point. The following code
  3556. * relies on the ability of the linker to provide the
  3557. * offset of a (static) per cpu variable into the per cpu area.
  3558. */
  3559. zone->pageset = &boot_pageset;
  3560. if (zone->present_pages)
  3561. printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n",
  3562. zone->name, zone->present_pages,
  3563. zone_batchsize(zone));
  3564. }
  3565. int __meminit init_currently_empty_zone(struct zone *zone,
  3566. unsigned long zone_start_pfn,
  3567. unsigned long size,
  3568. enum memmap_context context)
  3569. {
  3570. struct pglist_data *pgdat = zone->zone_pgdat;
  3571. int ret;
  3572. ret = zone_wait_table_init(zone, size);
  3573. if (ret)
  3574. return ret;
  3575. pgdat->nr_zones = zone_idx(zone) + 1;
  3576. zone->zone_start_pfn = zone_start_pfn;
  3577. mminit_dprintk(MMINIT_TRACE, "memmap_init",
  3578. "Initialising map node %d zone %lu pfns %lu -> %lu\n",
  3579. pgdat->node_id,
  3580. (unsigned long)zone_idx(zone),
  3581. zone_start_pfn, (zone_start_pfn + size));
  3582. zone_init_free_lists(zone);
  3583. return 0;
  3584. }
  3585. #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
  3586. #ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
  3587. /*
  3588. * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
  3589. * Architectures may implement their own version but if add_active_range()
  3590. * was used and there are no special requirements, this is a convenient
  3591. * alternative
  3592. */
  3593. int __meminit __early_pfn_to_nid(unsigned long pfn)
  3594. {
  3595. unsigned long start_pfn, end_pfn;
  3596. int i, nid;
  3597. for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
  3598. if (start_pfn <= pfn && pfn < end_pfn)
  3599. return nid;
  3600. /* This is a memory hole */
  3601. return -1;
  3602. }
  3603. #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
  3604. int __meminit early_pfn_to_nid(unsigned long pfn)
  3605. {
  3606. int nid;
  3607. nid = __early_pfn_to_nid(pfn);
  3608. if (nid >= 0)
  3609. return nid;
  3610. /* just returns 0 */
  3611. return 0;
  3612. }
  3613. #ifdef CONFIG_NODES_SPAN_OTHER_NODES
  3614. bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
  3615. {
  3616. int nid;
  3617. nid = __early_pfn_to_nid(pfn);
  3618. if (nid >= 0 && nid != node)
  3619. return false;
  3620. return true;
  3621. }
  3622. #endif
  3623. /**
  3624. * free_bootmem_with_active_regions - Call free_bootmem_node for each active range
  3625. * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed.
  3626. * @max_low_pfn: The highest PFN that will be passed to free_bootmem_node
  3627. *
  3628. * If an architecture guarantees that all ranges registered with
  3629. * add_active_ranges() contain no holes and may be freed, this
  3630. * this function may be used instead of calling free_bootmem() manually.
  3631. */
  3632. void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn)
  3633. {
  3634. unsigned long start_pfn, end_pfn;
  3635. int i, this_nid;
  3636. for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) {
  3637. start_pfn = min(start_pfn, max_low_pfn);
  3638. end_pfn = min(end_pfn, max_low_pfn);
  3639. if (start_pfn < end_pfn)
  3640. free_bootmem_node(NODE_DATA(this_nid),
  3641. PFN_PHYS(start_pfn),
  3642. (end_pfn - start_pfn) << PAGE_SHIFT);
  3643. }
  3644. }
  3645. /**
  3646. * sparse_memory_present_with_active_regions - Call memory_present for each active range
  3647. * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used.
  3648. *
  3649. * If an architecture guarantees that all ranges registered with
  3650. * add_active_ranges() contain no holes and may be freed, this
  3651. * function may be used instead of calling memory_present() manually.
  3652. */
  3653. void __init sparse_memory_present_with_active_regions(int nid)
  3654. {
  3655. unsigned long start_pfn, end_pfn;
  3656. int i, this_nid;
  3657. for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid)
  3658. memory_present(this_nid, start_pfn, end_pfn);
  3659. }
  3660. /**
  3661. * get_pfn_range_for_nid - Return the start and end page frames for a node
  3662. * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
  3663. * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
  3664. * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
  3665. *
  3666. * It returns the start and end page frame of a node based on information
  3667. * provided by an arch calling add_active_range(). If called for a node
  3668. * with no available memory, a warning is printed and the start and end
  3669. * PFNs will be 0.
  3670. */
  3671. void __meminit get_pfn_range_for_nid(unsigned int nid,
  3672. unsigned long *start_pfn, unsigned long *end_pfn)
  3673. {
  3674. unsigned long this_start_pfn, this_end_pfn;
  3675. int i;
  3676. *start_pfn = -1UL;
  3677. *end_pfn = 0;
  3678. for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) {
  3679. *start_pfn = min(*start_pfn, this_start_pfn);
  3680. *end_pfn = max(*end_pfn, this_end_pfn);
  3681. }
  3682. if (*start_pfn == -1UL)
  3683. *start_pfn = 0;
  3684. }
  3685. /*
  3686. * This finds a zone that can be used for ZONE_MOVABLE pages. The
  3687. * assumption is made that zones within a node are ordered in monotonic
  3688. * increasing memory addresses so that the "highest" populated zone is used
  3689. */
  3690. static void __init find_usable_zone_for_movable(void)
  3691. {
  3692. int zone_index;
  3693. for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
  3694. if (zone_index == ZONE_MOVABLE)
  3695. continue;
  3696. if (arch_zone_highest_possible_pfn[zone_index] >
  3697. arch_zone_lowest_possible_pfn[zone_index])
  3698. break;
  3699. }
  3700. VM_BUG_ON(zone_index == -1);
  3701. movable_zone = zone_index;
  3702. }
  3703. /*
  3704. * The zone ranges provided by the architecture do not include ZONE_MOVABLE
  3705. * because it is sized independent of architecture. Unlike the other zones,
  3706. * the starting point for ZONE_MOVABLE is not fixed. It may be different
  3707. * in each node depending on the size of each node and how evenly kernelcore
  3708. * is distributed. This helper function adjusts the zone ranges
  3709. * provided by the architecture for a given node by using the end of the
  3710. * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
  3711. * zones within a node are in order of monotonic increases memory addresses
  3712. */
  3713. static void __meminit adjust_zone_range_for_zone_movable(int nid,
  3714. unsigned long zone_type,
  3715. unsigned long node_start_pfn,
  3716. unsigned long node_end_pfn,
  3717. unsigned long *zone_start_pfn,
  3718. unsigned long *zone_end_pfn)
  3719. {
  3720. /* Only adjust if ZONE_MOVABLE is on this node */
  3721. if (zone_movable_pfn[nid]) {
  3722. /* Size ZONE_MOVABLE */
  3723. if (zone_type == ZONE_MOVABLE) {
  3724. *zone_start_pfn = zone_movable_pfn[nid];
  3725. *zone_end_pfn = min(node_end_pfn,
  3726. arch_zone_highest_possible_pfn[movable_zone]);
  3727. /* Adjust for ZONE_MOVABLE starting within this range */
  3728. } else if (*zone_start_pfn < zone_movable_pfn[nid] &&
  3729. *zone_end_pfn > zone_movable_pfn[nid]) {
  3730. *zone_end_pfn = zone_movable_pfn[nid];
  3731. /* Check if this whole range is within ZONE_MOVABLE */
  3732. } else if (*zone_start_pfn >= zone_movable_pfn[nid])
  3733. *zone_start_pfn = *zone_end_pfn;
  3734. }
  3735. }
  3736. /*
  3737. * Return the number of pages a zone spans in a node, including holes
  3738. * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
  3739. */
  3740. static unsigned long __meminit zone_spanned_pages_in_node(int nid,
  3741. unsigned long zone_type,
  3742. unsigned long *ignored)
  3743. {
  3744. unsigned long node_start_pfn, node_end_pfn;
  3745. unsigned long zone_start_pfn, zone_end_pfn;
  3746. /* Get the start and end of the node and zone */
  3747. get_pfn_range_for_nid(nid, &node_start_pfn, &node_end_pfn);
  3748. zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type];
  3749. zone_end_pfn = arch_zone_highest_possible_pfn[zone_type];
  3750. adjust_zone_range_for_zone_movable(nid, zone_type,
  3751. node_start_pfn, node_end_pfn,
  3752. &zone_start_pfn, &zone_end_pfn);
  3753. /* Check that this node has pages within the zone's required range */
  3754. if (zone_end_pfn < node_start_pfn || zone_start_pfn > node_end_pfn)
  3755. return 0;
  3756. /* Move the zone boundaries inside the node if necessary */
  3757. zone_end_pfn = min(zone_end_pfn, node_end_pfn);
  3758. zone_start_pfn = max(zone_start_pfn, node_start_pfn);
  3759. /* Return the spanned pages */
  3760. return zone_end_pfn - zone_start_pfn;
  3761. }
  3762. /*
  3763. * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
  3764. * then all holes in the requested range will be accounted for.
  3765. */
  3766. unsigned long __meminit __absent_pages_in_range(int nid,
  3767. unsigned long range_start_pfn,
  3768. unsigned long range_end_pfn)
  3769. {
  3770. unsigned long nr_absent = range_end_pfn - range_start_pfn;
  3771. unsigned long start_pfn, end_pfn;
  3772. int i;
  3773. for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
  3774. start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
  3775. end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
  3776. nr_absent -= end_pfn - start_pfn;
  3777. }
  3778. return nr_absent;
  3779. }
  3780. /**
  3781. * absent_pages_in_range - Return number of page frames in holes within a range
  3782. * @start_pfn: The start PFN to start searching for holes
  3783. * @end_pfn: The end PFN to stop searching for holes
  3784. *
  3785. * It returns the number of pages frames in memory holes within a range.
  3786. */
  3787. unsigned long __init absent_pages_in_range(unsigned long start_pfn,
  3788. unsigned long end_pfn)
  3789. {
  3790. return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
  3791. }
  3792. /* Return the number of page frames in holes in a zone on a node */
  3793. static unsigned long __meminit zone_absent_pages_in_node(int nid,
  3794. unsigned long zone_type,
  3795. unsigned long *ignored)
  3796. {
  3797. unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
  3798. unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
  3799. unsigned long node_start_pfn, node_end_pfn;
  3800. unsigned long zone_start_pfn, zone_end_pfn;
  3801. get_pfn_range_for_nid(nid, &node_start_pfn, &node_end_pfn);
  3802. zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
  3803. zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
  3804. adjust_zone_range_for_zone_movable(nid, zone_type,
  3805. node_start_pfn, node_end_pfn,
  3806. &zone_start_pfn, &zone_end_pfn);
  3807. return __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
  3808. }
  3809. /**
  3810. * sanitize_zone_movable_limit - Sanitize the zone_movable_limit array.
  3811. *
  3812. * zone_movable_limit is initialized as 0. This function will try to get
  3813. * the first ZONE_MOVABLE pfn of each node from movablemem_map, and
  3814. * assigne them to zone_movable_limit.
  3815. * zone_movable_limit[nid] == 0 means no limit for the node.
  3816. *
  3817. * Note: Each range is represented as [start_pfn, end_pfn)
  3818. */
  3819. static void __meminit sanitize_zone_movable_limit(void)
  3820. {
  3821. int map_pos = 0, i, nid;
  3822. unsigned long start_pfn, end_pfn;
  3823. if (!movablemem_map.nr_map)
  3824. return;
  3825. /* Iterate all ranges from minimum to maximum */
  3826. for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
  3827. /*
  3828. * If we have found lowest pfn of ZONE_MOVABLE of the node
  3829. * specified by user, just go on to check next range.
  3830. */
  3831. if (zone_movable_limit[nid])
  3832. continue;
  3833. #ifdef CONFIG_ZONE_DMA
  3834. /* Skip DMA memory. */
  3835. if (start_pfn < arch_zone_highest_possible_pfn[ZONE_DMA])
  3836. start_pfn = arch_zone_highest_possible_pfn[ZONE_DMA];
  3837. #endif
  3838. #ifdef CONFIG_ZONE_DMA32
  3839. /* Skip DMA32 memory. */
  3840. if (start_pfn < arch_zone_highest_possible_pfn[ZONE_DMA32])
  3841. start_pfn = arch_zone_highest_possible_pfn[ZONE_DMA32];
  3842. #endif
  3843. #ifdef CONFIG_HIGHMEM
  3844. /* Skip lowmem if ZONE_MOVABLE is highmem. */
  3845. if (zone_movable_is_highmem() &&
  3846. start_pfn < arch_zone_lowest_possible_pfn[ZONE_HIGHMEM])
  3847. start_pfn = arch_zone_lowest_possible_pfn[ZONE_HIGHMEM];
  3848. #endif
  3849. if (start_pfn >= end_pfn)
  3850. continue;
  3851. while (map_pos < movablemem_map.nr_map) {
  3852. if (end_pfn <= movablemem_map.map[map_pos].start_pfn)
  3853. break;
  3854. if (start_pfn >= movablemem_map.map[map_pos].end_pfn) {
  3855. map_pos++;
  3856. continue;
  3857. }
  3858. /*
  3859. * The start_pfn of ZONE_MOVABLE is either the minimum
  3860. * pfn specified by movablemem_map, or 0, which means
  3861. * the node has no ZONE_MOVABLE.
  3862. */
  3863. zone_movable_limit[nid] = max(start_pfn,
  3864. movablemem_map.map[map_pos].start_pfn);
  3865. break;
  3866. }
  3867. }
  3868. }
  3869. #else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
  3870. static inline unsigned long __meminit zone_spanned_pages_in_node(int nid,
  3871. unsigned long zone_type,
  3872. unsigned long *zones_size)
  3873. {
  3874. return zones_size[zone_type];
  3875. }
  3876. static inline unsigned long __meminit zone_absent_pages_in_node(int nid,
  3877. unsigned long zone_type,
  3878. unsigned long *zholes_size)
  3879. {
  3880. if (!zholes_size)
  3881. return 0;
  3882. return zholes_size[zone_type];
  3883. }
  3884. #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
  3885. static void __meminit calculate_node_totalpages(struct pglist_data *pgdat,
  3886. unsigned long *zones_size, unsigned long *zholes_size)
  3887. {
  3888. unsigned long realtotalpages, totalpages = 0;
  3889. enum zone_type i;
  3890. for (i = 0; i < MAX_NR_ZONES; i++)
  3891. totalpages += zone_spanned_pages_in_node(pgdat->node_id, i,
  3892. zones_size);
  3893. pgdat->node_spanned_pages = totalpages;
  3894. realtotalpages = totalpages;
  3895. for (i = 0; i < MAX_NR_ZONES; i++)
  3896. realtotalpages -=
  3897. zone_absent_pages_in_node(pgdat->node_id, i,
  3898. zholes_size);
  3899. pgdat->node_present_pages = realtotalpages;
  3900. printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id,
  3901. realtotalpages);
  3902. }
  3903. #ifndef CONFIG_SPARSEMEM
  3904. /*
  3905. * Calculate the size of the zone->blockflags rounded to an unsigned long
  3906. * Start by making sure zonesize is a multiple of pageblock_order by rounding
  3907. * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
  3908. * round what is now in bits to nearest long in bits, then return it in
  3909. * bytes.
  3910. */
  3911. static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize)
  3912. {
  3913. unsigned long usemapsize;
  3914. zonesize += zone_start_pfn & (pageblock_nr_pages-1);
  3915. usemapsize = roundup(zonesize, pageblock_nr_pages);
  3916. usemapsize = usemapsize >> pageblock_order;
  3917. usemapsize *= NR_PAGEBLOCK_BITS;
  3918. usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
  3919. return usemapsize / 8;
  3920. }
  3921. static void __init setup_usemap(struct pglist_data *pgdat,
  3922. struct zone *zone,
  3923. unsigned long zone_start_pfn,
  3924. unsigned long zonesize)
  3925. {
  3926. unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize);
  3927. zone->pageblock_flags = NULL;
  3928. if (usemapsize)
  3929. zone->pageblock_flags = alloc_bootmem_node_nopanic(pgdat,
  3930. usemapsize);
  3931. }
  3932. #else
  3933. static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone,
  3934. unsigned long zone_start_pfn, unsigned long zonesize) {}
  3935. #endif /* CONFIG_SPARSEMEM */
  3936. #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
  3937. /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
  3938. void __init set_pageblock_order(void)
  3939. {
  3940. unsigned int order;
  3941. /* Check that pageblock_nr_pages has not already been setup */
  3942. if (pageblock_order)
  3943. return;
  3944. if (HPAGE_SHIFT > PAGE_SHIFT)
  3945. order = HUGETLB_PAGE_ORDER;
  3946. else
  3947. order = MAX_ORDER - 1;
  3948. /*
  3949. * Assume the largest contiguous order of interest is a huge page.
  3950. * This value may be variable depending on boot parameters on IA64 and
  3951. * powerpc.
  3952. */
  3953. pageblock_order = order;
  3954. }
  3955. #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
  3956. /*
  3957. * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
  3958. * is unused as pageblock_order is set at compile-time. See
  3959. * include/linux/pageblock-flags.h for the values of pageblock_order based on
  3960. * the kernel config
  3961. */
  3962. void __init set_pageblock_order(void)
  3963. {
  3964. }
  3965. #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
  3966. static unsigned long __paginginit calc_memmap_size(unsigned long spanned_pages,
  3967. unsigned long present_pages)
  3968. {
  3969. unsigned long pages = spanned_pages;
  3970. /*
  3971. * Provide a more accurate estimation if there are holes within
  3972. * the zone and SPARSEMEM is in use. If there are holes within the
  3973. * zone, each populated memory region may cost us one or two extra
  3974. * memmap pages due to alignment because memmap pages for each
  3975. * populated regions may not naturally algined on page boundary.
  3976. * So the (present_pages >> 4) heuristic is a tradeoff for that.
  3977. */
  3978. if (spanned_pages > present_pages + (present_pages >> 4) &&
  3979. IS_ENABLED(CONFIG_SPARSEMEM))
  3980. pages = present_pages;
  3981. return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT;
  3982. }
  3983. /*
  3984. * Set up the zone data structures:
  3985. * - mark all pages reserved
  3986. * - mark all memory queues empty
  3987. * - clear the memory bitmaps
  3988. *
  3989. * NOTE: pgdat should get zeroed by caller.
  3990. */
  3991. static void __paginginit free_area_init_core(struct pglist_data *pgdat,
  3992. unsigned long *zones_size, unsigned long *zholes_size)
  3993. {
  3994. enum zone_type j;
  3995. int nid = pgdat->node_id;
  3996. unsigned long zone_start_pfn = pgdat->node_start_pfn;
  3997. int ret;
  3998. pgdat_resize_init(pgdat);
  3999. #ifdef CONFIG_NUMA_BALANCING
  4000. spin_lock_init(&pgdat->numabalancing_migrate_lock);
  4001. pgdat->numabalancing_migrate_nr_pages = 0;
  4002. pgdat->numabalancing_migrate_next_window = jiffies;
  4003. #endif
  4004. init_waitqueue_head(&pgdat->kswapd_wait);
  4005. init_waitqueue_head(&pgdat->pfmemalloc_wait);
  4006. pgdat_page_cgroup_init(pgdat);
  4007. for (j = 0; j < MAX_NR_ZONES; j++) {
  4008. struct zone *zone = pgdat->node_zones + j;
  4009. unsigned long size, realsize, freesize, memmap_pages;
  4010. size = zone_spanned_pages_in_node(nid, j, zones_size);
  4011. realsize = freesize = size - zone_absent_pages_in_node(nid, j,
  4012. zholes_size);
  4013. /*
  4014. * Adjust freesize so that it accounts for how much memory
  4015. * is used by this zone for memmap. This affects the watermark
  4016. * and per-cpu initialisations
  4017. */
  4018. memmap_pages = calc_memmap_size(size, realsize);
  4019. if (freesize >= memmap_pages) {
  4020. freesize -= memmap_pages;
  4021. if (memmap_pages)
  4022. printk(KERN_DEBUG
  4023. " %s zone: %lu pages used for memmap\n",
  4024. zone_names[j], memmap_pages);
  4025. } else
  4026. printk(KERN_WARNING
  4027. " %s zone: %lu pages exceeds freesize %lu\n",
  4028. zone_names[j], memmap_pages, freesize);
  4029. /* Account for reserved pages */
  4030. if (j == 0 && freesize > dma_reserve) {
  4031. freesize -= dma_reserve;
  4032. printk(KERN_DEBUG " %s zone: %lu pages reserved\n",
  4033. zone_names[0], dma_reserve);
  4034. }
  4035. if (!is_highmem_idx(j))
  4036. nr_kernel_pages += freesize;
  4037. /* Charge for highmem memmap if there are enough kernel pages */
  4038. else if (nr_kernel_pages > memmap_pages * 2)
  4039. nr_kernel_pages -= memmap_pages;
  4040. nr_all_pages += freesize;
  4041. zone->spanned_pages = size;
  4042. zone->present_pages = freesize;
  4043. /*
  4044. * Set an approximate value for lowmem here, it will be adjusted
  4045. * when the bootmem allocator frees pages into the buddy system.
  4046. * And all highmem pages will be managed by the buddy system.
  4047. */
  4048. zone->managed_pages = is_highmem_idx(j) ? realsize : freesize;
  4049. #ifdef CONFIG_NUMA
  4050. zone->node = nid;
  4051. zone->min_unmapped_pages = (freesize*sysctl_min_unmapped_ratio)
  4052. / 100;
  4053. zone->min_slab_pages = (freesize * sysctl_min_slab_ratio) / 100;
  4054. #endif
  4055. zone->name = zone_names[j];
  4056. spin_lock_init(&zone->lock);
  4057. spin_lock_init(&zone->lru_lock);
  4058. zone_seqlock_init(zone);
  4059. zone->zone_pgdat = pgdat;
  4060. zone_pcp_init(zone);
  4061. lruvec_init(&zone->lruvec);
  4062. if (!size)
  4063. continue;
  4064. set_pageblock_order();
  4065. setup_usemap(pgdat, zone, zone_start_pfn, size);
  4066. ret = init_currently_empty_zone(zone, zone_start_pfn,
  4067. size, MEMMAP_EARLY);
  4068. BUG_ON(ret);
  4069. memmap_init(size, nid, j, zone_start_pfn);
  4070. zone_start_pfn += size;
  4071. }
  4072. }
  4073. static void __init_refok alloc_node_mem_map(struct pglist_data *pgdat)
  4074. {
  4075. /* Skip empty nodes */
  4076. if (!pgdat->node_spanned_pages)
  4077. return;
  4078. #ifdef CONFIG_FLAT_NODE_MEM_MAP
  4079. /* ia64 gets its own node_mem_map, before this, without bootmem */
  4080. if (!pgdat->node_mem_map) {
  4081. unsigned long size, start, end;
  4082. struct page *map;
  4083. /*
  4084. * The zone's endpoints aren't required to be MAX_ORDER
  4085. * aligned but the node_mem_map endpoints must be in order
  4086. * for the buddy allocator to function correctly.
  4087. */
  4088. start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
  4089. end = pgdat->node_start_pfn + pgdat->node_spanned_pages;
  4090. end = ALIGN(end, MAX_ORDER_NR_PAGES);
  4091. size = (end - start) * sizeof(struct page);
  4092. map = alloc_remap(pgdat->node_id, size);
  4093. if (!map)
  4094. map = alloc_bootmem_node_nopanic(pgdat, size);
  4095. pgdat->node_mem_map = map + (pgdat->node_start_pfn - start);
  4096. }
  4097. #ifndef CONFIG_NEED_MULTIPLE_NODES
  4098. /*
  4099. * With no DISCONTIG, the global mem_map is just set as node 0's
  4100. */
  4101. if (pgdat == NODE_DATA(0)) {
  4102. mem_map = NODE_DATA(0)->node_mem_map;
  4103. #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
  4104. if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
  4105. mem_map -= (pgdat->node_start_pfn - ARCH_PFN_OFFSET);
  4106. #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
  4107. }
  4108. #endif
  4109. #endif /* CONFIG_FLAT_NODE_MEM_MAP */
  4110. }
  4111. void __paginginit free_area_init_node(int nid, unsigned long *zones_size,
  4112. unsigned long node_start_pfn, unsigned long *zholes_size)
  4113. {
  4114. pg_data_t *pgdat = NODE_DATA(nid);
  4115. /* pg_data_t should be reset to zero when it's allocated */
  4116. WARN_ON(pgdat->nr_zones || pgdat->classzone_idx);
  4117. pgdat->node_id = nid;
  4118. pgdat->node_start_pfn = node_start_pfn;
  4119. init_zone_allows_reclaim(nid);
  4120. calculate_node_totalpages(pgdat, zones_size, zholes_size);
  4121. alloc_node_mem_map(pgdat);
  4122. #ifdef CONFIG_FLAT_NODE_MEM_MAP
  4123. printk(KERN_DEBUG "free_area_init_node: node %d, pgdat %08lx, node_mem_map %08lx\n",
  4124. nid, (unsigned long)pgdat,
  4125. (unsigned long)pgdat->node_mem_map);
  4126. #endif
  4127. free_area_init_core(pgdat, zones_size, zholes_size);
  4128. }
  4129. #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
  4130. #if MAX_NUMNODES > 1
  4131. /*
  4132. * Figure out the number of possible node ids.
  4133. */
  4134. static void __init setup_nr_node_ids(void)
  4135. {
  4136. unsigned int node;
  4137. unsigned int highest = 0;
  4138. for_each_node_mask(node, node_possible_map)
  4139. highest = node;
  4140. nr_node_ids = highest + 1;
  4141. }
  4142. #else
  4143. static inline void setup_nr_node_ids(void)
  4144. {
  4145. }
  4146. #endif
  4147. /**
  4148. * node_map_pfn_alignment - determine the maximum internode alignment
  4149. *
  4150. * This function should be called after node map is populated and sorted.
  4151. * It calculates the maximum power of two alignment which can distinguish
  4152. * all the nodes.
  4153. *
  4154. * For example, if all nodes are 1GiB and aligned to 1GiB, the return value
  4155. * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the
  4156. * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is
  4157. * shifted, 1GiB is enough and this function will indicate so.
  4158. *
  4159. * This is used to test whether pfn -> nid mapping of the chosen memory
  4160. * model has fine enough granularity to avoid incorrect mapping for the
  4161. * populated node map.
  4162. *
  4163. * Returns the determined alignment in pfn's. 0 if there is no alignment
  4164. * requirement (single node).
  4165. */
  4166. unsigned long __init node_map_pfn_alignment(void)
  4167. {
  4168. unsigned long accl_mask = 0, last_end = 0;
  4169. unsigned long start, end, mask;
  4170. int last_nid = -1;
  4171. int i, nid;
  4172. for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {
  4173. if (!start || last_nid < 0 || last_nid == nid) {
  4174. last_nid = nid;
  4175. last_end = end;
  4176. continue;
  4177. }
  4178. /*
  4179. * Start with a mask granular enough to pin-point to the
  4180. * start pfn and tick off bits one-by-one until it becomes
  4181. * too coarse to separate the current node from the last.
  4182. */
  4183. mask = ~((1 << __ffs(start)) - 1);
  4184. while (mask && last_end <= (start & (mask << 1)))
  4185. mask <<= 1;
  4186. /* accumulate all internode masks */
  4187. accl_mask |= mask;
  4188. }
  4189. /* convert mask to number of pages */
  4190. return ~accl_mask + 1;
  4191. }
  4192. /* Find the lowest pfn for a node */
  4193. static unsigned long __init find_min_pfn_for_node(int nid)
  4194. {
  4195. unsigned long min_pfn = ULONG_MAX;
  4196. unsigned long start_pfn;
  4197. int i;
  4198. for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL)
  4199. min_pfn = min(min_pfn, start_pfn);
  4200. if (min_pfn == ULONG_MAX) {
  4201. printk(KERN_WARNING
  4202. "Could not find start_pfn for node %d\n", nid);
  4203. return 0;
  4204. }
  4205. return min_pfn;
  4206. }
  4207. /**
  4208. * find_min_pfn_with_active_regions - Find the minimum PFN registered
  4209. *
  4210. * It returns the minimum PFN based on information provided via
  4211. * add_active_range().
  4212. */
  4213. unsigned long __init find_min_pfn_with_active_regions(void)
  4214. {
  4215. return find_min_pfn_for_node(MAX_NUMNODES);
  4216. }
  4217. /*
  4218. * early_calculate_totalpages()
  4219. * Sum pages in active regions for movable zone.
  4220. * Populate N_MEMORY for calculating usable_nodes.
  4221. */
  4222. static unsigned long __init early_calculate_totalpages(void)
  4223. {
  4224. unsigned long totalpages = 0;
  4225. unsigned long start_pfn, end_pfn;
  4226. int i, nid;
  4227. for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
  4228. unsigned long pages = end_pfn - start_pfn;
  4229. totalpages += pages;
  4230. if (pages)
  4231. node_set_state(nid, N_MEMORY);
  4232. }
  4233. return totalpages;
  4234. }
  4235. /*
  4236. * Find the PFN the Movable zone begins in each node. Kernel memory
  4237. * is spread evenly between nodes as long as the nodes have enough
  4238. * memory. When they don't, some nodes will have more kernelcore than
  4239. * others
  4240. */
  4241. static void __init find_zone_movable_pfns_for_nodes(void)
  4242. {
  4243. int i, nid;
  4244. unsigned long usable_startpfn;
  4245. unsigned long kernelcore_node, kernelcore_remaining;
  4246. /* save the state before borrow the nodemask */
  4247. nodemask_t saved_node_state = node_states[N_MEMORY];
  4248. unsigned long totalpages = early_calculate_totalpages();
  4249. int usable_nodes = nodes_weight(node_states[N_MEMORY]);
  4250. /*
  4251. * If movablecore was specified, calculate what size of
  4252. * kernelcore that corresponds so that memory usable for
  4253. * any allocation type is evenly spread. If both kernelcore
  4254. * and movablecore are specified, then the value of kernelcore
  4255. * will be used for required_kernelcore if it's greater than
  4256. * what movablecore would have allowed.
  4257. */
  4258. if (required_movablecore) {
  4259. unsigned long corepages;
  4260. /*
  4261. * Round-up so that ZONE_MOVABLE is at least as large as what
  4262. * was requested by the user
  4263. */
  4264. required_movablecore =
  4265. roundup(required_movablecore, MAX_ORDER_NR_PAGES);
  4266. corepages = totalpages - required_movablecore;
  4267. required_kernelcore = max(required_kernelcore, corepages);
  4268. }
  4269. /*
  4270. * If neither kernelcore/movablecore nor movablemem_map is specified,
  4271. * there is no ZONE_MOVABLE. But if movablemem_map is specified, the
  4272. * start pfn of ZONE_MOVABLE has been stored in zone_movable_limit[].
  4273. */
  4274. if (!required_kernelcore) {
  4275. if (movablemem_map.nr_map)
  4276. memcpy(zone_movable_pfn, zone_movable_limit,
  4277. sizeof(zone_movable_pfn));
  4278. goto out;
  4279. }
  4280. /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
  4281. usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
  4282. restart:
  4283. /* Spread kernelcore memory as evenly as possible throughout nodes */
  4284. kernelcore_node = required_kernelcore / usable_nodes;
  4285. for_each_node_state(nid, N_MEMORY) {
  4286. unsigned long start_pfn, end_pfn;
  4287. /*
  4288. * Recalculate kernelcore_node if the division per node
  4289. * now exceeds what is necessary to satisfy the requested
  4290. * amount of memory for the kernel
  4291. */
  4292. if (required_kernelcore < kernelcore_node)
  4293. kernelcore_node = required_kernelcore / usable_nodes;
  4294. /*
  4295. * As the map is walked, we track how much memory is usable
  4296. * by the kernel using kernelcore_remaining. When it is
  4297. * 0, the rest of the node is usable by ZONE_MOVABLE
  4298. */
  4299. kernelcore_remaining = kernelcore_node;
  4300. /* Go through each range of PFNs within this node */
  4301. for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
  4302. unsigned long size_pages;
  4303. /*
  4304. * Find more memory for kernelcore in
  4305. * [zone_movable_pfn[nid], zone_movable_limit[nid]).
  4306. */
  4307. start_pfn = max(start_pfn, zone_movable_pfn[nid]);
  4308. if (start_pfn >= end_pfn)
  4309. continue;
  4310. if (zone_movable_limit[nid]) {
  4311. end_pfn = min(end_pfn, zone_movable_limit[nid]);
  4312. /* No range left for kernelcore in this node */
  4313. if (start_pfn >= end_pfn) {
  4314. zone_movable_pfn[nid] =
  4315. zone_movable_limit[nid];
  4316. break;
  4317. }
  4318. }
  4319. /* Account for what is only usable for kernelcore */
  4320. if (start_pfn < usable_startpfn) {
  4321. unsigned long kernel_pages;
  4322. kernel_pages = min(end_pfn, usable_startpfn)
  4323. - start_pfn;
  4324. kernelcore_remaining -= min(kernel_pages,
  4325. kernelcore_remaining);
  4326. required_kernelcore -= min(kernel_pages,
  4327. required_kernelcore);
  4328. /* Continue if range is now fully accounted */
  4329. if (end_pfn <= usable_startpfn) {
  4330. /*
  4331. * Push zone_movable_pfn to the end so
  4332. * that if we have to rebalance
  4333. * kernelcore across nodes, we will
  4334. * not double account here
  4335. */
  4336. zone_movable_pfn[nid] = end_pfn;
  4337. continue;
  4338. }
  4339. start_pfn = usable_startpfn;
  4340. }
  4341. /*
  4342. * The usable PFN range for ZONE_MOVABLE is from
  4343. * start_pfn->end_pfn. Calculate size_pages as the
  4344. * number of pages used as kernelcore
  4345. */
  4346. size_pages = end_pfn - start_pfn;
  4347. if (size_pages > kernelcore_remaining)
  4348. size_pages = kernelcore_remaining;
  4349. zone_movable_pfn[nid] = start_pfn + size_pages;
  4350. /*
  4351. * Some kernelcore has been met, update counts and
  4352. * break if the kernelcore for this node has been
  4353. * satisified
  4354. */
  4355. required_kernelcore -= min(required_kernelcore,
  4356. size_pages);
  4357. kernelcore_remaining -= size_pages;
  4358. if (!kernelcore_remaining)
  4359. break;
  4360. }
  4361. }
  4362. /*
  4363. * If there is still required_kernelcore, we do another pass with one
  4364. * less node in the count. This will push zone_movable_pfn[nid] further
  4365. * along on the nodes that still have memory until kernelcore is
  4366. * satisified
  4367. */
  4368. usable_nodes--;
  4369. if (usable_nodes && required_kernelcore > usable_nodes)
  4370. goto restart;
  4371. out:
  4372. /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
  4373. for (nid = 0; nid < MAX_NUMNODES; nid++)
  4374. zone_movable_pfn[nid] =
  4375. roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
  4376. /* restore the node_state */
  4377. node_states[N_MEMORY] = saved_node_state;
  4378. }
  4379. /* Any regular or high memory on that node ? */
  4380. static void check_for_memory(pg_data_t *pgdat, int nid)
  4381. {
  4382. enum zone_type zone_type;
  4383. if (N_MEMORY == N_NORMAL_MEMORY)
  4384. return;
  4385. for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) {
  4386. struct zone *zone = &pgdat->node_zones[zone_type];
  4387. if (zone->present_pages) {
  4388. node_set_state(nid, N_HIGH_MEMORY);
  4389. if (N_NORMAL_MEMORY != N_HIGH_MEMORY &&
  4390. zone_type <= ZONE_NORMAL)
  4391. node_set_state(nid, N_NORMAL_MEMORY);
  4392. break;
  4393. }
  4394. }
  4395. }
  4396. /**
  4397. * free_area_init_nodes - Initialise all pg_data_t and zone data
  4398. * @max_zone_pfn: an array of max PFNs for each zone
  4399. *
  4400. * This will call free_area_init_node() for each active node in the system.
  4401. * Using the page ranges provided by add_active_range(), the size of each
  4402. * zone in each node and their holes is calculated. If the maximum PFN
  4403. * between two adjacent zones match, it is assumed that the zone is empty.
  4404. * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
  4405. * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
  4406. * starts where the previous one ended. For example, ZONE_DMA32 starts
  4407. * at arch_max_dma_pfn.
  4408. */
  4409. void __init free_area_init_nodes(unsigned long *max_zone_pfn)
  4410. {
  4411. unsigned long start_pfn, end_pfn;
  4412. int i, nid;
  4413. /* Record where the zone boundaries are */
  4414. memset(arch_zone_lowest_possible_pfn, 0,
  4415. sizeof(arch_zone_lowest_possible_pfn));
  4416. memset(arch_zone_highest_possible_pfn, 0,
  4417. sizeof(arch_zone_highest_possible_pfn));
  4418. arch_zone_lowest_possible_pfn[0] = find_min_pfn_with_active_regions();
  4419. arch_zone_highest_possible_pfn[0] = max_zone_pfn[0];
  4420. for (i = 1; i < MAX_NR_ZONES; i++) {
  4421. if (i == ZONE_MOVABLE)
  4422. continue;
  4423. arch_zone_lowest_possible_pfn[i] =
  4424. arch_zone_highest_possible_pfn[i-1];
  4425. arch_zone_highest_possible_pfn[i] =
  4426. max(max_zone_pfn[i], arch_zone_lowest_possible_pfn[i]);
  4427. }
  4428. arch_zone_lowest_possible_pfn[ZONE_MOVABLE] = 0;
  4429. arch_zone_highest_possible_pfn[ZONE_MOVABLE] = 0;
  4430. /* Find the PFNs that ZONE_MOVABLE begins at in each node */
  4431. memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
  4432. find_usable_zone_for_movable();
  4433. sanitize_zone_movable_limit();
  4434. find_zone_movable_pfns_for_nodes();
  4435. /* Print out the zone ranges */
  4436. printk("Zone ranges:\n");
  4437. for (i = 0; i < MAX_NR_ZONES; i++) {
  4438. if (i == ZONE_MOVABLE)
  4439. continue;
  4440. printk(KERN_CONT " %-8s ", zone_names[i]);
  4441. if (arch_zone_lowest_possible_pfn[i] ==
  4442. arch_zone_highest_possible_pfn[i])
  4443. printk(KERN_CONT "empty\n");
  4444. else
  4445. printk(KERN_CONT "[mem %0#10lx-%0#10lx]\n",
  4446. arch_zone_lowest_possible_pfn[i] << PAGE_SHIFT,
  4447. (arch_zone_highest_possible_pfn[i]
  4448. << PAGE_SHIFT) - 1);
  4449. }
  4450. /* Print out the PFNs ZONE_MOVABLE begins at in each node */
  4451. printk("Movable zone start for each node\n");
  4452. for (i = 0; i < MAX_NUMNODES; i++) {
  4453. if (zone_movable_pfn[i])
  4454. printk(" Node %d: %#010lx\n", i,
  4455. zone_movable_pfn[i] << PAGE_SHIFT);
  4456. }
  4457. /* Print out the early node map */
  4458. printk("Early memory node ranges\n");
  4459. for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
  4460. printk(" node %3d: [mem %#010lx-%#010lx]\n", nid,
  4461. start_pfn << PAGE_SHIFT, (end_pfn << PAGE_SHIFT) - 1);
  4462. /* Initialise every node */
  4463. mminit_verify_pageflags_layout();
  4464. setup_nr_node_ids();
  4465. for_each_online_node(nid) {
  4466. pg_data_t *pgdat = NODE_DATA(nid);
  4467. free_area_init_node(nid, NULL,
  4468. find_min_pfn_for_node(nid), NULL);
  4469. /* Any memory on that node */
  4470. if (pgdat->node_present_pages)
  4471. node_set_state(nid, N_MEMORY);
  4472. check_for_memory(pgdat, nid);
  4473. }
  4474. }
  4475. static int __init cmdline_parse_core(char *p, unsigned long *core)
  4476. {
  4477. unsigned long long coremem;
  4478. if (!p)
  4479. return -EINVAL;
  4480. coremem = memparse(p, &p);
  4481. *core = coremem >> PAGE_SHIFT;
  4482. /* Paranoid check that UL is enough for the coremem value */
  4483. WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
  4484. return 0;
  4485. }
  4486. /*
  4487. * kernelcore=size sets the amount of memory for use for allocations that
  4488. * cannot be reclaimed or migrated.
  4489. */
  4490. static int __init cmdline_parse_kernelcore(char *p)
  4491. {
  4492. return cmdline_parse_core(p, &required_kernelcore);
  4493. }
  4494. /*
  4495. * movablecore=size sets the amount of memory for use for allocations that
  4496. * can be reclaimed or migrated.
  4497. */
  4498. static int __init cmdline_parse_movablecore(char *p)
  4499. {
  4500. return cmdline_parse_core(p, &required_movablecore);
  4501. }
  4502. early_param("kernelcore", cmdline_parse_kernelcore);
  4503. early_param("movablecore", cmdline_parse_movablecore);
  4504. /**
  4505. * insert_movablemem_map - Insert a memory range in to movablemem_map.map.
  4506. * @start_pfn: start pfn of the range
  4507. * @end_pfn: end pfn of the range
  4508. *
  4509. * This function will also merge the overlapped ranges, and sort the array
  4510. * by start_pfn in monotonic increasing order.
  4511. */
  4512. static void __init insert_movablemem_map(unsigned long start_pfn,
  4513. unsigned long end_pfn)
  4514. {
  4515. int pos, overlap;
  4516. /*
  4517. * pos will be at the 1st overlapped range, or the position
  4518. * where the element should be inserted.
  4519. */
  4520. for (pos = 0; pos < movablemem_map.nr_map; pos++)
  4521. if (start_pfn <= movablemem_map.map[pos].end_pfn)
  4522. break;
  4523. /* If there is no overlapped range, just insert the element. */
  4524. if (pos == movablemem_map.nr_map ||
  4525. end_pfn < movablemem_map.map[pos].start_pfn) {
  4526. /*
  4527. * If pos is not the end of array, we need to move all
  4528. * the rest elements backward.
  4529. */
  4530. if (pos < movablemem_map.nr_map)
  4531. memmove(&movablemem_map.map[pos+1],
  4532. &movablemem_map.map[pos],
  4533. sizeof(struct movablemem_entry) *
  4534. (movablemem_map.nr_map - pos));
  4535. movablemem_map.map[pos].start_pfn = start_pfn;
  4536. movablemem_map.map[pos].end_pfn = end_pfn;
  4537. movablemem_map.nr_map++;
  4538. return;
  4539. }
  4540. /* overlap will be at the last overlapped range */
  4541. for (overlap = pos + 1; overlap < movablemem_map.nr_map; overlap++)
  4542. if (end_pfn < movablemem_map.map[overlap].start_pfn)
  4543. break;
  4544. /*
  4545. * If there are more ranges overlapped, we need to merge them,
  4546. * and move the rest elements forward.
  4547. */
  4548. overlap--;
  4549. movablemem_map.map[pos].start_pfn = min(start_pfn,
  4550. movablemem_map.map[pos].start_pfn);
  4551. movablemem_map.map[pos].end_pfn = max(end_pfn,
  4552. movablemem_map.map[overlap].end_pfn);
  4553. if (pos != overlap && overlap + 1 != movablemem_map.nr_map)
  4554. memmove(&movablemem_map.map[pos+1],
  4555. &movablemem_map.map[overlap+1],
  4556. sizeof(struct movablemem_entry) *
  4557. (movablemem_map.nr_map - overlap - 1));
  4558. movablemem_map.nr_map -= overlap - pos;
  4559. }
  4560. /**
  4561. * movablemem_map_add_region - Add a memory range into movablemem_map.
  4562. * @start: physical start address of range
  4563. * @end: physical end address of range
  4564. *
  4565. * This function transform the physical address into pfn, and then add the
  4566. * range into movablemem_map by calling insert_movablemem_map().
  4567. */
  4568. static void __init movablemem_map_add_region(u64 start, u64 size)
  4569. {
  4570. unsigned long start_pfn, end_pfn;
  4571. /* In case size == 0 or start + size overflows */
  4572. if (start + size <= start)
  4573. return;
  4574. if (movablemem_map.nr_map >= ARRAY_SIZE(movablemem_map.map)) {
  4575. pr_err("movablemem_map: too many entries;"
  4576. " ignoring [mem %#010llx-%#010llx]\n",
  4577. (unsigned long long) start,
  4578. (unsigned long long) (start + size - 1));
  4579. return;
  4580. }
  4581. start_pfn = PFN_DOWN(start);
  4582. end_pfn = PFN_UP(start + size);
  4583. insert_movablemem_map(start_pfn, end_pfn);
  4584. }
  4585. /*
  4586. * cmdline_parse_movablemem_map - Parse boot option movablemem_map.
  4587. * @p: The boot option of the following format:
  4588. * movablemem_map=nn[KMG]@ss[KMG]
  4589. *
  4590. * This option sets the memory range [ss, ss+nn) to be used as movable memory.
  4591. *
  4592. * Return: 0 on success or -EINVAL on failure.
  4593. */
  4594. static int __init cmdline_parse_movablemem_map(char *p)
  4595. {
  4596. char *oldp;
  4597. u64 start_at, mem_size;
  4598. if (!p)
  4599. goto err;
  4600. oldp = p;
  4601. mem_size = memparse(p, &p);
  4602. if (p == oldp)
  4603. goto err;
  4604. if (*p == '@') {
  4605. oldp = ++p;
  4606. start_at = memparse(p, &p);
  4607. if (p == oldp || *p != '\0')
  4608. goto err;
  4609. movablemem_map_add_region(start_at, mem_size);
  4610. return 0;
  4611. }
  4612. err:
  4613. return -EINVAL;
  4614. }
  4615. early_param("movablemem_map", cmdline_parse_movablemem_map);
  4616. #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
  4617. /**
  4618. * set_dma_reserve - set the specified number of pages reserved in the first zone
  4619. * @new_dma_reserve: The number of pages to mark reserved
  4620. *
  4621. * The per-cpu batchsize and zone watermarks are determined by present_pages.
  4622. * In the DMA zone, a significant percentage may be consumed by kernel image
  4623. * and other unfreeable allocations which can skew the watermarks badly. This
  4624. * function may optionally be used to account for unfreeable pages in the
  4625. * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
  4626. * smaller per-cpu batchsize.
  4627. */
  4628. void __init set_dma_reserve(unsigned long new_dma_reserve)
  4629. {
  4630. dma_reserve = new_dma_reserve;
  4631. }
  4632. void __init free_area_init(unsigned long *zones_size)
  4633. {
  4634. free_area_init_node(0, zones_size,
  4635. __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL);
  4636. }
  4637. static int page_alloc_cpu_notify(struct notifier_block *self,
  4638. unsigned long action, void *hcpu)
  4639. {
  4640. int cpu = (unsigned long)hcpu;
  4641. if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
  4642. lru_add_drain_cpu(cpu);
  4643. drain_pages(cpu);
  4644. /*
  4645. * Spill the event counters of the dead processor
  4646. * into the current processors event counters.
  4647. * This artificially elevates the count of the current
  4648. * processor.
  4649. */
  4650. vm_events_fold_cpu(cpu);
  4651. /*
  4652. * Zero the differential counters of the dead processor
  4653. * so that the vm statistics are consistent.
  4654. *
  4655. * This is only okay since the processor is dead and cannot
  4656. * race with what we are doing.
  4657. */
  4658. refresh_cpu_vm_stats(cpu);
  4659. }
  4660. return NOTIFY_OK;
  4661. }
  4662. void __init page_alloc_init(void)
  4663. {
  4664. hotcpu_notifier(page_alloc_cpu_notify, 0);
  4665. }
  4666. /*
  4667. * calculate_totalreserve_pages - called when sysctl_lower_zone_reserve_ratio
  4668. * or min_free_kbytes changes.
  4669. */
  4670. static void calculate_totalreserve_pages(void)
  4671. {
  4672. struct pglist_data *pgdat;
  4673. unsigned long reserve_pages = 0;
  4674. enum zone_type i, j;
  4675. for_each_online_pgdat(pgdat) {
  4676. for (i = 0; i < MAX_NR_ZONES; i++) {
  4677. struct zone *zone = pgdat->node_zones + i;
  4678. unsigned long max = 0;
  4679. /* Find valid and maximum lowmem_reserve in the zone */
  4680. for (j = i; j < MAX_NR_ZONES; j++) {
  4681. if (zone->lowmem_reserve[j] > max)
  4682. max = zone->lowmem_reserve[j];
  4683. }
  4684. /* we treat the high watermark as reserved pages. */
  4685. max += high_wmark_pages(zone);
  4686. if (max > zone->present_pages)
  4687. max = zone->present_pages;
  4688. reserve_pages += max;
  4689. /*
  4690. * Lowmem reserves are not available to
  4691. * GFP_HIGHUSER page cache allocations and
  4692. * kswapd tries to balance zones to their high
  4693. * watermark. As a result, neither should be
  4694. * regarded as dirtyable memory, to prevent a
  4695. * situation where reclaim has to clean pages
  4696. * in order to balance the zones.
  4697. */
  4698. zone->dirty_balance_reserve = max;
  4699. }
  4700. }
  4701. dirty_balance_reserve = reserve_pages;
  4702. totalreserve_pages = reserve_pages;
  4703. }
  4704. /*
  4705. * setup_per_zone_lowmem_reserve - called whenever
  4706. * sysctl_lower_zone_reserve_ratio changes. Ensures that each zone
  4707. * has a correct pages reserved value, so an adequate number of
  4708. * pages are left in the zone after a successful __alloc_pages().
  4709. */
  4710. static void setup_per_zone_lowmem_reserve(void)
  4711. {
  4712. struct pglist_data *pgdat;
  4713. enum zone_type j, idx;
  4714. for_each_online_pgdat(pgdat) {
  4715. for (j = 0; j < MAX_NR_ZONES; j++) {
  4716. struct zone *zone = pgdat->node_zones + j;
  4717. unsigned long present_pages = zone->present_pages;
  4718. zone->lowmem_reserve[j] = 0;
  4719. idx = j;
  4720. while (idx) {
  4721. struct zone *lower_zone;
  4722. idx--;
  4723. if (sysctl_lowmem_reserve_ratio[idx] < 1)
  4724. sysctl_lowmem_reserve_ratio[idx] = 1;
  4725. lower_zone = pgdat->node_zones + idx;
  4726. lower_zone->lowmem_reserve[j] = present_pages /
  4727. sysctl_lowmem_reserve_ratio[idx];
  4728. present_pages += lower_zone->present_pages;
  4729. }
  4730. }
  4731. }
  4732. /* update totalreserve_pages */
  4733. calculate_totalreserve_pages();
  4734. }
  4735. static void __setup_per_zone_wmarks(void)
  4736. {
  4737. unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
  4738. unsigned long lowmem_pages = 0;
  4739. struct zone *zone;
  4740. unsigned long flags;
  4741. /* Calculate total number of !ZONE_HIGHMEM pages */
  4742. for_each_zone(zone) {
  4743. if (!is_highmem(zone))
  4744. lowmem_pages += zone->present_pages;
  4745. }
  4746. for_each_zone(zone) {
  4747. u64 tmp;
  4748. spin_lock_irqsave(&zone->lock, flags);
  4749. tmp = (u64)pages_min * zone->present_pages;
  4750. do_div(tmp, lowmem_pages);
  4751. if (is_highmem(zone)) {
  4752. /*
  4753. * __GFP_HIGH and PF_MEMALLOC allocations usually don't
  4754. * need highmem pages, so cap pages_min to a small
  4755. * value here.
  4756. *
  4757. * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
  4758. * deltas controls asynch page reclaim, and so should
  4759. * not be capped for highmem.
  4760. */
  4761. unsigned long min_pages;
  4762. min_pages = zone->present_pages / 1024;
  4763. min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL);
  4764. zone->watermark[WMARK_MIN] = min_pages;
  4765. } else {
  4766. /*
  4767. * If it's a lowmem zone, reserve a number of pages
  4768. * proportionate to the zone's size.
  4769. */
  4770. zone->watermark[WMARK_MIN] = tmp;
  4771. }
  4772. zone->watermark[WMARK_LOW] = min_wmark_pages(zone) + (tmp >> 2);
  4773. zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + (tmp >> 1);
  4774. setup_zone_migrate_reserve(zone);
  4775. spin_unlock_irqrestore(&zone->lock, flags);
  4776. }
  4777. /* update totalreserve_pages */
  4778. calculate_totalreserve_pages();
  4779. }
  4780. /**
  4781. * setup_per_zone_wmarks - called when min_free_kbytes changes
  4782. * or when memory is hot-{added|removed}
  4783. *
  4784. * Ensures that the watermark[min,low,high] values for each zone are set
  4785. * correctly with respect to min_free_kbytes.
  4786. */
  4787. void setup_per_zone_wmarks(void)
  4788. {
  4789. mutex_lock(&zonelists_mutex);
  4790. __setup_per_zone_wmarks();
  4791. mutex_unlock(&zonelists_mutex);
  4792. }
  4793. /*
  4794. * The inactive anon list should be small enough that the VM never has to
  4795. * do too much work, but large enough that each inactive page has a chance
  4796. * to be referenced again before it is swapped out.
  4797. *
  4798. * The inactive_anon ratio is the target ratio of ACTIVE_ANON to
  4799. * INACTIVE_ANON pages on this zone's LRU, maintained by the
  4800. * pageout code. A zone->inactive_ratio of 3 means 3:1 or 25% of
  4801. * the anonymous pages are kept on the inactive list.
  4802. *
  4803. * total target max
  4804. * memory ratio inactive anon
  4805. * -------------------------------------
  4806. * 10MB 1 5MB
  4807. * 100MB 1 50MB
  4808. * 1GB 3 250MB
  4809. * 10GB 10 0.9GB
  4810. * 100GB 31 3GB
  4811. * 1TB 101 10GB
  4812. * 10TB 320 32GB
  4813. */
  4814. static void __meminit calculate_zone_inactive_ratio(struct zone *zone)
  4815. {
  4816. unsigned int gb, ratio;
  4817. /* Zone size in gigabytes */
  4818. gb = zone->present_pages >> (30 - PAGE_SHIFT);
  4819. if (gb)
  4820. ratio = int_sqrt(10 * gb);
  4821. else
  4822. ratio = 1;
  4823. zone->inactive_ratio = ratio;
  4824. }
  4825. static void __meminit setup_per_zone_inactive_ratio(void)
  4826. {
  4827. struct zone *zone;
  4828. for_each_zone(zone)
  4829. calculate_zone_inactive_ratio(zone);
  4830. }
  4831. /*
  4832. * Initialise min_free_kbytes.
  4833. *
  4834. * For small machines we want it small (128k min). For large machines
  4835. * we want it large (64MB max). But it is not linear, because network
  4836. * bandwidth does not increase linearly with machine size. We use
  4837. *
  4838. * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy:
  4839. * min_free_kbytes = sqrt(lowmem_kbytes * 16)
  4840. *
  4841. * which yields
  4842. *
  4843. * 16MB: 512k
  4844. * 32MB: 724k
  4845. * 64MB: 1024k
  4846. * 128MB: 1448k
  4847. * 256MB: 2048k
  4848. * 512MB: 2896k
  4849. * 1024MB: 4096k
  4850. * 2048MB: 5792k
  4851. * 4096MB: 8192k
  4852. * 8192MB: 11584k
  4853. * 16384MB: 16384k
  4854. */
  4855. int __meminit init_per_zone_wmark_min(void)
  4856. {
  4857. unsigned long lowmem_kbytes;
  4858. lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
  4859. min_free_kbytes = int_sqrt(lowmem_kbytes * 16);
  4860. if (min_free_kbytes < 128)
  4861. min_free_kbytes = 128;
  4862. if (min_free_kbytes > 65536)
  4863. min_free_kbytes = 65536;
  4864. setup_per_zone_wmarks();
  4865. refresh_zone_stat_thresholds();
  4866. setup_per_zone_lowmem_reserve();
  4867. setup_per_zone_inactive_ratio();
  4868. return 0;
  4869. }
  4870. module_init(init_per_zone_wmark_min)
  4871. /*
  4872. * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so
  4873. * that we can call two helper functions whenever min_free_kbytes
  4874. * changes.
  4875. */
  4876. int min_free_kbytes_sysctl_handler(ctl_table *table, int write,
  4877. void __user *buffer, size_t *length, loff_t *ppos)
  4878. {
  4879. proc_dointvec(table, write, buffer, length, ppos);
  4880. if (write)
  4881. setup_per_zone_wmarks();
  4882. return 0;
  4883. }
  4884. #ifdef CONFIG_NUMA
  4885. int sysctl_min_unmapped_ratio_sysctl_handler(ctl_table *table, int write,
  4886. void __user *buffer, size_t *length, loff_t *ppos)
  4887. {
  4888. struct zone *zone;
  4889. int rc;
  4890. rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
  4891. if (rc)
  4892. return rc;
  4893. for_each_zone(zone)
  4894. zone->min_unmapped_pages = (zone->present_pages *
  4895. sysctl_min_unmapped_ratio) / 100;
  4896. return 0;
  4897. }
  4898. int sysctl_min_slab_ratio_sysctl_handler(ctl_table *table, int write,
  4899. void __user *buffer, size_t *length, loff_t *ppos)
  4900. {
  4901. struct zone *zone;
  4902. int rc;
  4903. rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
  4904. if (rc)
  4905. return rc;
  4906. for_each_zone(zone)
  4907. zone->min_slab_pages = (zone->present_pages *
  4908. sysctl_min_slab_ratio) / 100;
  4909. return 0;
  4910. }
  4911. #endif
  4912. /*
  4913. * lowmem_reserve_ratio_sysctl_handler - just a wrapper around
  4914. * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve()
  4915. * whenever sysctl_lowmem_reserve_ratio changes.
  4916. *
  4917. * The reserve ratio obviously has absolutely no relation with the
  4918. * minimum watermarks. The lowmem reserve ratio can only make sense
  4919. * if in function of the boot time zone sizes.
  4920. */
  4921. int lowmem_reserve_ratio_sysctl_handler(ctl_table *table, int write,
  4922. void __user *buffer, size_t *length, loff_t *ppos)
  4923. {
  4924. proc_dointvec_minmax(table, write, buffer, length, ppos);
  4925. setup_per_zone_lowmem_reserve();
  4926. return 0;
  4927. }
  4928. /*
  4929. * percpu_pagelist_fraction - changes the pcp->high for each zone on each
  4930. * cpu. It is the fraction of total pages in each zone that a hot per cpu pagelist
  4931. * can have before it gets flushed back to buddy allocator.
  4932. */
  4933. int percpu_pagelist_fraction_sysctl_handler(ctl_table *table, int write,
  4934. void __user *buffer, size_t *length, loff_t *ppos)
  4935. {
  4936. struct zone *zone;
  4937. unsigned int cpu;
  4938. int ret;
  4939. ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
  4940. if (!write || (ret < 0))
  4941. return ret;
  4942. for_each_populated_zone(zone) {
  4943. for_each_possible_cpu(cpu) {
  4944. unsigned long high;
  4945. high = zone->present_pages / percpu_pagelist_fraction;
  4946. setup_pagelist_highmark(
  4947. per_cpu_ptr(zone->pageset, cpu), high);
  4948. }
  4949. }
  4950. return 0;
  4951. }
  4952. int hashdist = HASHDIST_DEFAULT;
  4953. #ifdef CONFIG_NUMA
  4954. static int __init set_hashdist(char *str)
  4955. {
  4956. if (!str)
  4957. return 0;
  4958. hashdist = simple_strtoul(str, &str, 0);
  4959. return 1;
  4960. }
  4961. __setup("hashdist=", set_hashdist);
  4962. #endif
  4963. /*
  4964. * allocate a large system hash table from bootmem
  4965. * - it is assumed that the hash table must contain an exact power-of-2
  4966. * quantity of entries
  4967. * - limit is the number of hash buckets, not the total allocation size
  4968. */
  4969. void *__init alloc_large_system_hash(const char *tablename,
  4970. unsigned long bucketsize,
  4971. unsigned long numentries,
  4972. int scale,
  4973. int flags,
  4974. unsigned int *_hash_shift,
  4975. unsigned int *_hash_mask,
  4976. unsigned long low_limit,
  4977. unsigned long high_limit)
  4978. {
  4979. unsigned long long max = high_limit;
  4980. unsigned long log2qty, size;
  4981. void *table = NULL;
  4982. /* allow the kernel cmdline to have a say */
  4983. if (!numentries) {
  4984. /* round applicable memory size up to nearest megabyte */
  4985. numentries = nr_kernel_pages;
  4986. numentries += (1UL << (20 - PAGE_SHIFT)) - 1;
  4987. numentries >>= 20 - PAGE_SHIFT;
  4988. numentries <<= 20 - PAGE_SHIFT;
  4989. /* limit to 1 bucket per 2^scale bytes of low memory */
  4990. if (scale > PAGE_SHIFT)
  4991. numentries >>= (scale - PAGE_SHIFT);
  4992. else
  4993. numentries <<= (PAGE_SHIFT - scale);
  4994. /* Make sure we've got at least a 0-order allocation.. */
  4995. if (unlikely(flags & HASH_SMALL)) {
  4996. /* Makes no sense without HASH_EARLY */
  4997. WARN_ON(!(flags & HASH_EARLY));
  4998. if (!(numentries >> *_hash_shift)) {
  4999. numentries = 1UL << *_hash_shift;
  5000. BUG_ON(!numentries);
  5001. }
  5002. } else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
  5003. numentries = PAGE_SIZE / bucketsize;
  5004. }
  5005. numentries = roundup_pow_of_two(numentries);
  5006. /* limit allocation size to 1/16 total memory by default */
  5007. if (max == 0) {
  5008. max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
  5009. do_div(max, bucketsize);
  5010. }
  5011. max = min(max, 0x80000000ULL);
  5012. if (numentries < low_limit)
  5013. numentries = low_limit;
  5014. if (numentries > max)
  5015. numentries = max;
  5016. log2qty = ilog2(numentries);
  5017. do {
  5018. size = bucketsize << log2qty;
  5019. if (flags & HASH_EARLY)
  5020. table = alloc_bootmem_nopanic(size);
  5021. else if (hashdist)
  5022. table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL);
  5023. else {
  5024. /*
  5025. * If bucketsize is not a power-of-two, we may free
  5026. * some pages at the end of hash table which
  5027. * alloc_pages_exact() automatically does
  5028. */
  5029. if (get_order(size) < MAX_ORDER) {
  5030. table = alloc_pages_exact(size, GFP_ATOMIC);
  5031. kmemleak_alloc(table, size, 1, GFP_ATOMIC);
  5032. }
  5033. }
  5034. } while (!table && size > PAGE_SIZE && --log2qty);
  5035. if (!table)
  5036. panic("Failed to allocate %s hash table\n", tablename);
  5037. printk(KERN_INFO "%s hash table entries: %ld (order: %d, %lu bytes)\n",
  5038. tablename,
  5039. (1UL << log2qty),
  5040. ilog2(size) - PAGE_SHIFT,
  5041. size);
  5042. if (_hash_shift)
  5043. *_hash_shift = log2qty;
  5044. if (_hash_mask)
  5045. *_hash_mask = (1 << log2qty) - 1;
  5046. return table;
  5047. }
  5048. /* Return a pointer to the bitmap storing bits affecting a block of pages */
  5049. static inline unsigned long *get_pageblock_bitmap(struct zone *zone,
  5050. unsigned long pfn)
  5051. {
  5052. #ifdef CONFIG_SPARSEMEM
  5053. return __pfn_to_section(pfn)->pageblock_flags;
  5054. #else
  5055. return zone->pageblock_flags;
  5056. #endif /* CONFIG_SPARSEMEM */
  5057. }
  5058. static inline int pfn_to_bitidx(struct zone *zone, unsigned long pfn)
  5059. {
  5060. #ifdef CONFIG_SPARSEMEM
  5061. pfn &= (PAGES_PER_SECTION-1);
  5062. return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
  5063. #else
  5064. pfn = pfn - round_down(zone->zone_start_pfn, pageblock_nr_pages);
  5065. return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
  5066. #endif /* CONFIG_SPARSEMEM */
  5067. }
  5068. /**
  5069. * get_pageblock_flags_group - Return the requested group of flags for the pageblock_nr_pages block of pages
  5070. * @page: The page within the block of interest
  5071. * @start_bitidx: The first bit of interest to retrieve
  5072. * @end_bitidx: The last bit of interest
  5073. * returns pageblock_bits flags
  5074. */
  5075. unsigned long get_pageblock_flags_group(struct page *page,
  5076. int start_bitidx, int end_bitidx)
  5077. {
  5078. struct zone *zone;
  5079. unsigned long *bitmap;
  5080. unsigned long pfn, bitidx;
  5081. unsigned long flags = 0;
  5082. unsigned long value = 1;
  5083. zone = page_zone(page);
  5084. pfn = page_to_pfn(page);
  5085. bitmap = get_pageblock_bitmap(zone, pfn);
  5086. bitidx = pfn_to_bitidx(zone, pfn);
  5087. for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1)
  5088. if (test_bit(bitidx + start_bitidx, bitmap))
  5089. flags |= value;
  5090. return flags;
  5091. }
  5092. /**
  5093. * set_pageblock_flags_group - Set the requested group of flags for a pageblock_nr_pages block of pages
  5094. * @page: The page within the block of interest
  5095. * @start_bitidx: The first bit of interest
  5096. * @end_bitidx: The last bit of interest
  5097. * @flags: The flags to set
  5098. */
  5099. void set_pageblock_flags_group(struct page *page, unsigned long flags,
  5100. int start_bitidx, int end_bitidx)
  5101. {
  5102. struct zone *zone;
  5103. unsigned long *bitmap;
  5104. unsigned long pfn, bitidx;
  5105. unsigned long value = 1;
  5106. zone = page_zone(page);
  5107. pfn = page_to_pfn(page);
  5108. bitmap = get_pageblock_bitmap(zone, pfn);
  5109. bitidx = pfn_to_bitidx(zone, pfn);
  5110. VM_BUG_ON(pfn < zone->zone_start_pfn);
  5111. VM_BUG_ON(pfn >= zone->zone_start_pfn + zone->spanned_pages);
  5112. for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1)
  5113. if (flags & value)
  5114. __set_bit(bitidx + start_bitidx, bitmap);
  5115. else
  5116. __clear_bit(bitidx + start_bitidx, bitmap);
  5117. }
  5118. /*
  5119. * This function checks whether pageblock includes unmovable pages or not.
  5120. * If @count is not zero, it is okay to include less @count unmovable pages
  5121. *
  5122. * PageLRU check wihtout isolation or lru_lock could race so that
  5123. * MIGRATE_MOVABLE block might include unmovable pages. It means you can't
  5124. * expect this function should be exact.
  5125. */
  5126. bool has_unmovable_pages(struct zone *zone, struct page *page, int count,
  5127. bool skip_hwpoisoned_pages)
  5128. {
  5129. unsigned long pfn, iter, found;
  5130. int mt;
  5131. /*
  5132. * For avoiding noise data, lru_add_drain_all() should be called
  5133. * If ZONE_MOVABLE, the zone never contains unmovable pages
  5134. */
  5135. if (zone_idx(zone) == ZONE_MOVABLE)
  5136. return false;
  5137. mt = get_pageblock_migratetype(page);
  5138. if (mt == MIGRATE_MOVABLE || is_migrate_cma(mt))
  5139. return false;
  5140. pfn = page_to_pfn(page);
  5141. for (found = 0, iter = 0; iter < pageblock_nr_pages; iter++) {
  5142. unsigned long check = pfn + iter;
  5143. if (!pfn_valid_within(check))
  5144. continue;
  5145. page = pfn_to_page(check);
  5146. /*
  5147. * We can't use page_count without pin a page
  5148. * because another CPU can free compound page.
  5149. * This check already skips compound tails of THP
  5150. * because their page->_count is zero at all time.
  5151. */
  5152. if (!atomic_read(&page->_count)) {
  5153. if (PageBuddy(page))
  5154. iter += (1 << page_order(page)) - 1;
  5155. continue;
  5156. }
  5157. /*
  5158. * The HWPoisoned page may be not in buddy system, and
  5159. * page_count() is not 0.
  5160. */
  5161. if (skip_hwpoisoned_pages && PageHWPoison(page))
  5162. continue;
  5163. if (!PageLRU(page))
  5164. found++;
  5165. /*
  5166. * If there are RECLAIMABLE pages, we need to check it.
  5167. * But now, memory offline itself doesn't call shrink_slab()
  5168. * and it still to be fixed.
  5169. */
  5170. /*
  5171. * If the page is not RAM, page_count()should be 0.
  5172. * we don't need more check. This is an _used_ not-movable page.
  5173. *
  5174. * The problematic thing here is PG_reserved pages. PG_reserved
  5175. * is set to both of a memory hole page and a _used_ kernel
  5176. * page at boot.
  5177. */
  5178. if (found > count)
  5179. return true;
  5180. }
  5181. return false;
  5182. }
  5183. bool is_pageblock_removable_nolock(struct page *page)
  5184. {
  5185. struct zone *zone;
  5186. unsigned long pfn;
  5187. /*
  5188. * We have to be careful here because we are iterating over memory
  5189. * sections which are not zone aware so we might end up outside of
  5190. * the zone but still within the section.
  5191. * We have to take care about the node as well. If the node is offline
  5192. * its NODE_DATA will be NULL - see page_zone.
  5193. */
  5194. if (!node_online(page_to_nid(page)))
  5195. return false;
  5196. zone = page_zone(page);
  5197. pfn = page_to_pfn(page);
  5198. if (zone->zone_start_pfn > pfn ||
  5199. zone->zone_start_pfn + zone->spanned_pages <= pfn)
  5200. return false;
  5201. return !has_unmovable_pages(zone, page, 0, true);
  5202. }
  5203. #ifdef CONFIG_CMA
  5204. static unsigned long pfn_max_align_down(unsigned long pfn)
  5205. {
  5206. return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES,
  5207. pageblock_nr_pages) - 1);
  5208. }
  5209. static unsigned long pfn_max_align_up(unsigned long pfn)
  5210. {
  5211. return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES,
  5212. pageblock_nr_pages));
  5213. }
  5214. /* [start, end) must belong to a single zone. */
  5215. static int __alloc_contig_migrate_range(struct compact_control *cc,
  5216. unsigned long start, unsigned long end)
  5217. {
  5218. /* This function is based on compact_zone() from compaction.c. */
  5219. unsigned long nr_reclaimed;
  5220. unsigned long pfn = start;
  5221. unsigned int tries = 0;
  5222. int ret = 0;
  5223. migrate_prep();
  5224. while (pfn < end || !list_empty(&cc->migratepages)) {
  5225. if (fatal_signal_pending(current)) {
  5226. ret = -EINTR;
  5227. break;
  5228. }
  5229. if (list_empty(&cc->migratepages)) {
  5230. cc->nr_migratepages = 0;
  5231. pfn = isolate_migratepages_range(cc->zone, cc,
  5232. pfn, end, true);
  5233. if (!pfn) {
  5234. ret = -EINTR;
  5235. break;
  5236. }
  5237. tries = 0;
  5238. } else if (++tries == 5) {
  5239. ret = ret < 0 ? ret : -EBUSY;
  5240. break;
  5241. }
  5242. nr_reclaimed = reclaim_clean_pages_from_list(cc->zone,
  5243. &cc->migratepages);
  5244. cc->nr_migratepages -= nr_reclaimed;
  5245. ret = migrate_pages(&cc->migratepages,
  5246. alloc_migrate_target,
  5247. 0, false, MIGRATE_SYNC,
  5248. MR_CMA);
  5249. }
  5250. if (ret < 0) {
  5251. putback_movable_pages(&cc->migratepages);
  5252. return ret;
  5253. }
  5254. return 0;
  5255. }
  5256. /**
  5257. * alloc_contig_range() -- tries to allocate given range of pages
  5258. * @start: start PFN to allocate
  5259. * @end: one-past-the-last PFN to allocate
  5260. * @migratetype: migratetype of the underlaying pageblocks (either
  5261. * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks
  5262. * in range must have the same migratetype and it must
  5263. * be either of the two.
  5264. *
  5265. * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES
  5266. * aligned, however it's the caller's responsibility to guarantee that
  5267. * we are the only thread that changes migrate type of pageblocks the
  5268. * pages fall in.
  5269. *
  5270. * The PFN range must belong to a single zone.
  5271. *
  5272. * Returns zero on success or negative error code. On success all
  5273. * pages which PFN is in [start, end) are allocated for the caller and
  5274. * need to be freed with free_contig_range().
  5275. */
  5276. int alloc_contig_range(unsigned long start, unsigned long end,
  5277. unsigned migratetype)
  5278. {
  5279. unsigned long outer_start, outer_end;
  5280. int ret = 0, order;
  5281. struct compact_control cc = {
  5282. .nr_migratepages = 0,
  5283. .order = -1,
  5284. .zone = page_zone(pfn_to_page(start)),
  5285. .sync = true,
  5286. .ignore_skip_hint = true,
  5287. };
  5288. INIT_LIST_HEAD(&cc.migratepages);
  5289. /*
  5290. * What we do here is we mark all pageblocks in range as
  5291. * MIGRATE_ISOLATE. Because pageblock and max order pages may
  5292. * have different sizes, and due to the way page allocator
  5293. * work, we align the range to biggest of the two pages so
  5294. * that page allocator won't try to merge buddies from
  5295. * different pageblocks and change MIGRATE_ISOLATE to some
  5296. * other migration type.
  5297. *
  5298. * Once the pageblocks are marked as MIGRATE_ISOLATE, we
  5299. * migrate the pages from an unaligned range (ie. pages that
  5300. * we are interested in). This will put all the pages in
  5301. * range back to page allocator as MIGRATE_ISOLATE.
  5302. *
  5303. * When this is done, we take the pages in range from page
  5304. * allocator removing them from the buddy system. This way
  5305. * page allocator will never consider using them.
  5306. *
  5307. * This lets us mark the pageblocks back as
  5308. * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the
  5309. * aligned range but not in the unaligned, original range are
  5310. * put back to page allocator so that buddy can use them.
  5311. */
  5312. ret = start_isolate_page_range(pfn_max_align_down(start),
  5313. pfn_max_align_up(end), migratetype,
  5314. false);
  5315. if (ret)
  5316. return ret;
  5317. ret = __alloc_contig_migrate_range(&cc, start, end);
  5318. if (ret)
  5319. goto done;
  5320. /*
  5321. * Pages from [start, end) are within a MAX_ORDER_NR_PAGES
  5322. * aligned blocks that are marked as MIGRATE_ISOLATE. What's
  5323. * more, all pages in [start, end) are free in page allocator.
  5324. * What we are going to do is to allocate all pages from
  5325. * [start, end) (that is remove them from page allocator).
  5326. *
  5327. * The only problem is that pages at the beginning and at the
  5328. * end of interesting range may be not aligned with pages that
  5329. * page allocator holds, ie. they can be part of higher order
  5330. * pages. Because of this, we reserve the bigger range and
  5331. * once this is done free the pages we are not interested in.
  5332. *
  5333. * We don't have to hold zone->lock here because the pages are
  5334. * isolated thus they won't get removed from buddy.
  5335. */
  5336. lru_add_drain_all();
  5337. drain_all_pages();
  5338. order = 0;
  5339. outer_start = start;
  5340. while (!PageBuddy(pfn_to_page(outer_start))) {
  5341. if (++order >= MAX_ORDER) {
  5342. ret = -EBUSY;
  5343. goto done;
  5344. }
  5345. outer_start &= ~0UL << order;
  5346. }
  5347. /* Make sure the range is really isolated. */
  5348. if (test_pages_isolated(outer_start, end, false)) {
  5349. pr_warn("alloc_contig_range test_pages_isolated(%lx, %lx) failed\n",
  5350. outer_start, end);
  5351. ret = -EBUSY;
  5352. goto done;
  5353. }
  5354. /* Grab isolated pages from freelists. */
  5355. outer_end = isolate_freepages_range(&cc, outer_start, end);
  5356. if (!outer_end) {
  5357. ret = -EBUSY;
  5358. goto done;
  5359. }
  5360. /* Free head and tail (if any) */
  5361. if (start != outer_start)
  5362. free_contig_range(outer_start, start - outer_start);
  5363. if (end != outer_end)
  5364. free_contig_range(end, outer_end - end);
  5365. done:
  5366. undo_isolate_page_range(pfn_max_align_down(start),
  5367. pfn_max_align_up(end), migratetype);
  5368. return ret;
  5369. }
  5370. void free_contig_range(unsigned long pfn, unsigned nr_pages)
  5371. {
  5372. unsigned int count = 0;
  5373. for (; nr_pages--; pfn++) {
  5374. struct page *page = pfn_to_page(pfn);
  5375. count += page_count(page) != 1;
  5376. __free_page(page);
  5377. }
  5378. WARN(count != 0, "%d pages are still in use!\n", count);
  5379. }
  5380. #endif
  5381. #ifdef CONFIG_MEMORY_HOTPLUG
  5382. static int __meminit __zone_pcp_update(void *data)
  5383. {
  5384. struct zone *zone = data;
  5385. int cpu;
  5386. unsigned long batch = zone_batchsize(zone), flags;
  5387. for_each_possible_cpu(cpu) {
  5388. struct per_cpu_pageset *pset;
  5389. struct per_cpu_pages *pcp;
  5390. pset = per_cpu_ptr(zone->pageset, cpu);
  5391. pcp = &pset->pcp;
  5392. local_irq_save(flags);
  5393. if (pcp->count > 0)
  5394. free_pcppages_bulk(zone, pcp->count, pcp);
  5395. drain_zonestat(zone, pset);
  5396. setup_pageset(pset, batch);
  5397. local_irq_restore(flags);
  5398. }
  5399. return 0;
  5400. }
  5401. void __meminit zone_pcp_update(struct zone *zone)
  5402. {
  5403. stop_machine(__zone_pcp_update, zone, NULL);
  5404. }
  5405. #endif
  5406. void zone_pcp_reset(struct zone *zone)
  5407. {
  5408. unsigned long flags;
  5409. int cpu;
  5410. struct per_cpu_pageset *pset;
  5411. /* avoid races with drain_pages() */
  5412. local_irq_save(flags);
  5413. if (zone->pageset != &boot_pageset) {
  5414. for_each_online_cpu(cpu) {
  5415. pset = per_cpu_ptr(zone->pageset, cpu);
  5416. drain_zonestat(zone, pset);
  5417. }
  5418. free_percpu(zone->pageset);
  5419. zone->pageset = &boot_pageset;
  5420. }
  5421. local_irq_restore(flags);
  5422. }
  5423. #ifdef CONFIG_MEMORY_HOTREMOVE
  5424. /*
  5425. * All pages in the range must be isolated before calling this.
  5426. */
  5427. void
  5428. __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
  5429. {
  5430. struct page *page;
  5431. struct zone *zone;
  5432. int order, i;
  5433. unsigned long pfn;
  5434. unsigned long flags;
  5435. /* find the first valid pfn */
  5436. for (pfn = start_pfn; pfn < end_pfn; pfn++)
  5437. if (pfn_valid(pfn))
  5438. break;
  5439. if (pfn == end_pfn)
  5440. return;
  5441. zone = page_zone(pfn_to_page(pfn));
  5442. spin_lock_irqsave(&zone->lock, flags);
  5443. pfn = start_pfn;
  5444. while (pfn < end_pfn) {
  5445. if (!pfn_valid(pfn)) {
  5446. pfn++;
  5447. continue;
  5448. }
  5449. page = pfn_to_page(pfn);
  5450. /*
  5451. * The HWPoisoned page may be not in buddy system, and
  5452. * page_count() is not 0.
  5453. */
  5454. if (unlikely(!PageBuddy(page) && PageHWPoison(page))) {
  5455. pfn++;
  5456. SetPageReserved(page);
  5457. continue;
  5458. }
  5459. BUG_ON(page_count(page));
  5460. BUG_ON(!PageBuddy(page));
  5461. order = page_order(page);
  5462. #ifdef CONFIG_DEBUG_VM
  5463. printk(KERN_INFO "remove from free list %lx %d %lx\n",
  5464. pfn, 1 << order, end_pfn);
  5465. #endif
  5466. list_del(&page->lru);
  5467. rmv_page_order(page);
  5468. zone->free_area[order].nr_free--;
  5469. for (i = 0; i < (1 << order); i++)
  5470. SetPageReserved((page+i));
  5471. pfn += (1 << order);
  5472. }
  5473. spin_unlock_irqrestore(&zone->lock, flags);
  5474. }
  5475. #endif
  5476. #ifdef CONFIG_MEMORY_FAILURE
  5477. bool is_free_buddy_page(struct page *page)
  5478. {
  5479. struct zone *zone = page_zone(page);
  5480. unsigned long pfn = page_to_pfn(page);
  5481. unsigned long flags;
  5482. int order;
  5483. spin_lock_irqsave(&zone->lock, flags);
  5484. for (order = 0; order < MAX_ORDER; order++) {
  5485. struct page *page_head = page - (pfn & ((1 << order) - 1));
  5486. if (PageBuddy(page_head) && page_order(page_head) >= order)
  5487. break;
  5488. }
  5489. spin_unlock_irqrestore(&zone->lock, flags);
  5490. return order < MAX_ORDER;
  5491. }
  5492. #endif
  5493. static const struct trace_print_flags pageflag_names[] = {
  5494. {1UL << PG_locked, "locked" },
  5495. {1UL << PG_error, "error" },
  5496. {1UL << PG_referenced, "referenced" },
  5497. {1UL << PG_uptodate, "uptodate" },
  5498. {1UL << PG_dirty, "dirty" },
  5499. {1UL << PG_lru, "lru" },
  5500. {1UL << PG_active, "active" },
  5501. {1UL << PG_slab, "slab" },
  5502. {1UL << PG_owner_priv_1, "owner_priv_1" },
  5503. {1UL << PG_arch_1, "arch_1" },
  5504. {1UL << PG_reserved, "reserved" },
  5505. {1UL << PG_private, "private" },
  5506. {1UL << PG_private_2, "private_2" },
  5507. {1UL << PG_writeback, "writeback" },
  5508. #ifdef CONFIG_PAGEFLAGS_EXTENDED
  5509. {1UL << PG_head, "head" },
  5510. {1UL << PG_tail, "tail" },
  5511. #else
  5512. {1UL << PG_compound, "compound" },
  5513. #endif
  5514. {1UL << PG_swapcache, "swapcache" },
  5515. {1UL << PG_mappedtodisk, "mappedtodisk" },
  5516. {1UL << PG_reclaim, "reclaim" },
  5517. {1UL << PG_swapbacked, "swapbacked" },
  5518. {1UL << PG_unevictable, "unevictable" },
  5519. #ifdef CONFIG_MMU
  5520. {1UL << PG_mlocked, "mlocked" },
  5521. #endif
  5522. #ifdef CONFIG_ARCH_USES_PG_UNCACHED
  5523. {1UL << PG_uncached, "uncached" },
  5524. #endif
  5525. #ifdef CONFIG_MEMORY_FAILURE
  5526. {1UL << PG_hwpoison, "hwpoison" },
  5527. #endif
  5528. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  5529. {1UL << PG_compound_lock, "compound_lock" },
  5530. #endif
  5531. };
  5532. static void dump_page_flags(unsigned long flags)
  5533. {
  5534. const char *delim = "";
  5535. unsigned long mask;
  5536. int i;
  5537. BUILD_BUG_ON(ARRAY_SIZE(pageflag_names) != __NR_PAGEFLAGS);
  5538. printk(KERN_ALERT "page flags: %#lx(", flags);
  5539. /* remove zone id */
  5540. flags &= (1UL << NR_PAGEFLAGS) - 1;
  5541. for (i = 0; i < ARRAY_SIZE(pageflag_names) && flags; i++) {
  5542. mask = pageflag_names[i].mask;
  5543. if ((flags & mask) != mask)
  5544. continue;
  5545. flags &= ~mask;
  5546. printk("%s%s", delim, pageflag_names[i].name);
  5547. delim = "|";
  5548. }
  5549. /* check for left over flags */
  5550. if (flags)
  5551. printk("%s%#lx", delim, flags);
  5552. printk(")\n");
  5553. }
  5554. void dump_page(struct page *page)
  5555. {
  5556. printk(KERN_ALERT
  5557. "page:%p count:%d mapcount:%d mapping:%p index:%#lx\n",
  5558. page, atomic_read(&page->_count), page_mapcount(page),
  5559. page->mapping, page->index);
  5560. dump_page_flags(page->flags);
  5561. mem_cgroup_print_bad_page(page);
  5562. }