page_alloc.c 168 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106
  1. /*
  2. * linux/mm/page_alloc.c
  3. *
  4. * Manages the free list, the system allocates free pages here.
  5. * Note that kmalloc() lives in slab.c
  6. *
  7. * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
  8. * Swap reorganised 29.12.95, Stephen Tweedie
  9. * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
  10. * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
  11. * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
  12. * Zone balancing, Kanoj Sarcar, SGI, Jan 2000
  13. * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
  14. * (lots of bits borrowed from Ingo Molnar & Andrew Morton)
  15. */
  16. #include <linux/stddef.h>
  17. #include <linux/mm.h>
  18. #include <linux/swap.h>
  19. #include <linux/interrupt.h>
  20. #include <linux/pagemap.h>
  21. #include <linux/jiffies.h>
  22. #include <linux/bootmem.h>
  23. #include <linux/memblock.h>
  24. #include <linux/compiler.h>
  25. #include <linux/kernel.h>
  26. #include <linux/kmemcheck.h>
  27. #include <linux/module.h>
  28. #include <linux/suspend.h>
  29. #include <linux/pagevec.h>
  30. #include <linux/blkdev.h>
  31. #include <linux/slab.h>
  32. #include <linux/ratelimit.h>
  33. #include <linux/oom.h>
  34. #include <linux/notifier.h>
  35. #include <linux/topology.h>
  36. #include <linux/sysctl.h>
  37. #include <linux/cpu.h>
  38. #include <linux/cpuset.h>
  39. #include <linux/memory_hotplug.h>
  40. #include <linux/nodemask.h>
  41. #include <linux/vmalloc.h>
  42. #include <linux/vmstat.h>
  43. #include <linux/mempolicy.h>
  44. #include <linux/stop_machine.h>
  45. #include <linux/sort.h>
  46. #include <linux/pfn.h>
  47. #include <linux/backing-dev.h>
  48. #include <linux/fault-inject.h>
  49. #include <linux/page-isolation.h>
  50. #include <linux/page_cgroup.h>
  51. #include <linux/debugobjects.h>
  52. #include <linux/kmemleak.h>
  53. #include <linux/compaction.h>
  54. #include <trace/events/kmem.h>
  55. #include <linux/ftrace_event.h>
  56. #include <linux/memcontrol.h>
  57. #include <linux/prefetch.h>
  58. #include <linux/migrate.h>
  59. #include <linux/page-debug-flags.h>
  60. #include <asm/tlbflush.h>
  61. #include <asm/div64.h>
  62. #include "internal.h"
  63. #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
  64. DEFINE_PER_CPU(int, numa_node);
  65. EXPORT_PER_CPU_SYMBOL(numa_node);
  66. #endif
  67. #ifdef CONFIG_HAVE_MEMORYLESS_NODES
  68. /*
  69. * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
  70. * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
  71. * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
  72. * defined in <linux/topology.h>.
  73. */
  74. DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */
  75. EXPORT_PER_CPU_SYMBOL(_numa_mem_);
  76. #endif
  77. /*
  78. * Array of node states.
  79. */
  80. nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
  81. [N_POSSIBLE] = NODE_MASK_ALL,
  82. [N_ONLINE] = { { [0] = 1UL } },
  83. #ifndef CONFIG_NUMA
  84. [N_NORMAL_MEMORY] = { { [0] = 1UL } },
  85. #ifdef CONFIG_HIGHMEM
  86. [N_HIGH_MEMORY] = { { [0] = 1UL } },
  87. #endif
  88. [N_CPU] = { { [0] = 1UL } },
  89. #endif /* NUMA */
  90. };
  91. EXPORT_SYMBOL(node_states);
  92. unsigned long totalram_pages __read_mostly;
  93. unsigned long totalreserve_pages __read_mostly;
  94. /*
  95. * When calculating the number of globally allowed dirty pages, there
  96. * is a certain number of per-zone reserves that should not be
  97. * considered dirtyable memory. This is the sum of those reserves
  98. * over all existing zones that contribute dirtyable memory.
  99. */
  100. unsigned long dirty_balance_reserve __read_mostly;
  101. int percpu_pagelist_fraction;
  102. gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
  103. #ifdef CONFIG_PM_SLEEP
  104. /*
  105. * The following functions are used by the suspend/hibernate code to temporarily
  106. * change gfp_allowed_mask in order to avoid using I/O during memory allocations
  107. * while devices are suspended. To avoid races with the suspend/hibernate code,
  108. * they should always be called with pm_mutex held (gfp_allowed_mask also should
  109. * only be modified with pm_mutex held, unless the suspend/hibernate code is
  110. * guaranteed not to run in parallel with that modification).
  111. */
  112. static gfp_t saved_gfp_mask;
  113. void pm_restore_gfp_mask(void)
  114. {
  115. WARN_ON(!mutex_is_locked(&pm_mutex));
  116. if (saved_gfp_mask) {
  117. gfp_allowed_mask = saved_gfp_mask;
  118. saved_gfp_mask = 0;
  119. }
  120. }
  121. void pm_restrict_gfp_mask(void)
  122. {
  123. WARN_ON(!mutex_is_locked(&pm_mutex));
  124. WARN_ON(saved_gfp_mask);
  125. saved_gfp_mask = gfp_allowed_mask;
  126. gfp_allowed_mask &= ~GFP_IOFS;
  127. }
  128. bool pm_suspended_storage(void)
  129. {
  130. if ((gfp_allowed_mask & GFP_IOFS) == GFP_IOFS)
  131. return false;
  132. return true;
  133. }
  134. #endif /* CONFIG_PM_SLEEP */
  135. #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
  136. int pageblock_order __read_mostly;
  137. #endif
  138. static void __free_pages_ok(struct page *page, unsigned int order);
  139. /*
  140. * results with 256, 32 in the lowmem_reserve sysctl:
  141. * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
  142. * 1G machine -> (16M dma, 784M normal, 224M high)
  143. * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
  144. * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
  145. * HIGHMEM allocation will (224M+784M)/256 of ram reserved in ZONE_DMA
  146. *
  147. * TBD: should special case ZONE_DMA32 machines here - in those we normally
  148. * don't need any ZONE_NORMAL reservation
  149. */
  150. int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = {
  151. #ifdef CONFIG_ZONE_DMA
  152. 256,
  153. #endif
  154. #ifdef CONFIG_ZONE_DMA32
  155. 256,
  156. #endif
  157. #ifdef CONFIG_HIGHMEM
  158. 32,
  159. #endif
  160. 32,
  161. };
  162. EXPORT_SYMBOL(totalram_pages);
  163. static char * const zone_names[MAX_NR_ZONES] = {
  164. #ifdef CONFIG_ZONE_DMA
  165. "DMA",
  166. #endif
  167. #ifdef CONFIG_ZONE_DMA32
  168. "DMA32",
  169. #endif
  170. "Normal",
  171. #ifdef CONFIG_HIGHMEM
  172. "HighMem",
  173. #endif
  174. "Movable",
  175. };
  176. int min_free_kbytes = 1024;
  177. static unsigned long __meminitdata nr_kernel_pages;
  178. static unsigned long __meminitdata nr_all_pages;
  179. static unsigned long __meminitdata dma_reserve;
  180. #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
  181. static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES];
  182. static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES];
  183. static unsigned long __initdata required_kernelcore;
  184. static unsigned long __initdata required_movablecore;
  185. static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES];
  186. /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
  187. int movable_zone;
  188. EXPORT_SYMBOL(movable_zone);
  189. #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
  190. #if MAX_NUMNODES > 1
  191. int nr_node_ids __read_mostly = MAX_NUMNODES;
  192. int nr_online_nodes __read_mostly = 1;
  193. EXPORT_SYMBOL(nr_node_ids);
  194. EXPORT_SYMBOL(nr_online_nodes);
  195. #endif
  196. int page_group_by_mobility_disabled __read_mostly;
  197. /*
  198. * NOTE:
  199. * Don't use set_pageblock_migratetype(page, MIGRATE_ISOLATE) directly.
  200. * Instead, use {un}set_pageblock_isolate.
  201. */
  202. void set_pageblock_migratetype(struct page *page, int migratetype)
  203. {
  204. if (unlikely(page_group_by_mobility_disabled))
  205. migratetype = MIGRATE_UNMOVABLE;
  206. set_pageblock_flags_group(page, (unsigned long)migratetype,
  207. PB_migrate, PB_migrate_end);
  208. }
  209. bool oom_killer_disabled __read_mostly;
  210. #ifdef CONFIG_DEBUG_VM
  211. static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
  212. {
  213. int ret = 0;
  214. unsigned seq;
  215. unsigned long pfn = page_to_pfn(page);
  216. do {
  217. seq = zone_span_seqbegin(zone);
  218. if (pfn >= zone->zone_start_pfn + zone->spanned_pages)
  219. ret = 1;
  220. else if (pfn < zone->zone_start_pfn)
  221. ret = 1;
  222. } while (zone_span_seqretry(zone, seq));
  223. return ret;
  224. }
  225. static int page_is_consistent(struct zone *zone, struct page *page)
  226. {
  227. if (!pfn_valid_within(page_to_pfn(page)))
  228. return 0;
  229. if (zone != page_zone(page))
  230. return 0;
  231. return 1;
  232. }
  233. /*
  234. * Temporary debugging check for pages not lying within a given zone.
  235. */
  236. static int bad_range(struct zone *zone, struct page *page)
  237. {
  238. if (page_outside_zone_boundaries(zone, page))
  239. return 1;
  240. if (!page_is_consistent(zone, page))
  241. return 1;
  242. return 0;
  243. }
  244. #else
  245. static inline int bad_range(struct zone *zone, struct page *page)
  246. {
  247. return 0;
  248. }
  249. #endif
  250. static void bad_page(struct page *page)
  251. {
  252. static unsigned long resume;
  253. static unsigned long nr_shown;
  254. static unsigned long nr_unshown;
  255. /* Don't complain about poisoned pages */
  256. if (PageHWPoison(page)) {
  257. reset_page_mapcount(page); /* remove PageBuddy */
  258. return;
  259. }
  260. /*
  261. * Allow a burst of 60 reports, then keep quiet for that minute;
  262. * or allow a steady drip of one report per second.
  263. */
  264. if (nr_shown == 60) {
  265. if (time_before(jiffies, resume)) {
  266. nr_unshown++;
  267. goto out;
  268. }
  269. if (nr_unshown) {
  270. printk(KERN_ALERT
  271. "BUG: Bad page state: %lu messages suppressed\n",
  272. nr_unshown);
  273. nr_unshown = 0;
  274. }
  275. nr_shown = 0;
  276. }
  277. if (nr_shown++ == 0)
  278. resume = jiffies + 60 * HZ;
  279. printk(KERN_ALERT "BUG: Bad page state in process %s pfn:%05lx\n",
  280. current->comm, page_to_pfn(page));
  281. dump_page(page);
  282. print_modules();
  283. dump_stack();
  284. out:
  285. /* Leave bad fields for debug, except PageBuddy could make trouble */
  286. reset_page_mapcount(page); /* remove PageBuddy */
  287. add_taint(TAINT_BAD_PAGE);
  288. }
  289. /*
  290. * Higher-order pages are called "compound pages". They are structured thusly:
  291. *
  292. * The first PAGE_SIZE page is called the "head page".
  293. *
  294. * The remaining PAGE_SIZE pages are called "tail pages".
  295. *
  296. * All pages have PG_compound set. All tail pages have their ->first_page
  297. * pointing at the head page.
  298. *
  299. * The first tail page's ->lru.next holds the address of the compound page's
  300. * put_page() function. Its ->lru.prev holds the order of allocation.
  301. * This usage means that zero-order pages may not be compound.
  302. */
  303. static void free_compound_page(struct page *page)
  304. {
  305. __free_pages_ok(page, compound_order(page));
  306. }
  307. void prep_compound_page(struct page *page, unsigned long order)
  308. {
  309. int i;
  310. int nr_pages = 1 << order;
  311. set_compound_page_dtor(page, free_compound_page);
  312. set_compound_order(page, order);
  313. __SetPageHead(page);
  314. for (i = 1; i < nr_pages; i++) {
  315. struct page *p = page + i;
  316. __SetPageTail(p);
  317. set_page_count(p, 0);
  318. p->first_page = page;
  319. }
  320. }
  321. /* update __split_huge_page_refcount if you change this function */
  322. static int destroy_compound_page(struct page *page, unsigned long order)
  323. {
  324. int i;
  325. int nr_pages = 1 << order;
  326. int bad = 0;
  327. if (unlikely(compound_order(page) != order) ||
  328. unlikely(!PageHead(page))) {
  329. bad_page(page);
  330. bad++;
  331. }
  332. __ClearPageHead(page);
  333. for (i = 1; i < nr_pages; i++) {
  334. struct page *p = page + i;
  335. if (unlikely(!PageTail(p) || (p->first_page != page))) {
  336. bad_page(page);
  337. bad++;
  338. }
  339. __ClearPageTail(p);
  340. }
  341. return bad;
  342. }
  343. static inline void prep_zero_page(struct page *page, int order, gfp_t gfp_flags)
  344. {
  345. int i;
  346. /*
  347. * clear_highpage() will use KM_USER0, so it's a bug to use __GFP_ZERO
  348. * and __GFP_HIGHMEM from hard or soft interrupt context.
  349. */
  350. VM_BUG_ON((gfp_flags & __GFP_HIGHMEM) && in_interrupt());
  351. for (i = 0; i < (1 << order); i++)
  352. clear_highpage(page + i);
  353. }
  354. #ifdef CONFIG_DEBUG_PAGEALLOC
  355. unsigned int _debug_guardpage_minorder;
  356. static int __init debug_guardpage_minorder_setup(char *buf)
  357. {
  358. unsigned long res;
  359. if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) {
  360. printk(KERN_ERR "Bad debug_guardpage_minorder value\n");
  361. return 0;
  362. }
  363. _debug_guardpage_minorder = res;
  364. printk(KERN_INFO "Setting debug_guardpage_minorder to %lu\n", res);
  365. return 0;
  366. }
  367. __setup("debug_guardpage_minorder=", debug_guardpage_minorder_setup);
  368. static inline void set_page_guard_flag(struct page *page)
  369. {
  370. __set_bit(PAGE_DEBUG_FLAG_GUARD, &page->debug_flags);
  371. }
  372. static inline void clear_page_guard_flag(struct page *page)
  373. {
  374. __clear_bit(PAGE_DEBUG_FLAG_GUARD, &page->debug_flags);
  375. }
  376. #else
  377. static inline void set_page_guard_flag(struct page *page) { }
  378. static inline void clear_page_guard_flag(struct page *page) { }
  379. #endif
  380. static inline void set_page_order(struct page *page, int order)
  381. {
  382. set_page_private(page, order);
  383. __SetPageBuddy(page);
  384. }
  385. static inline void rmv_page_order(struct page *page)
  386. {
  387. __ClearPageBuddy(page);
  388. set_page_private(page, 0);
  389. }
  390. /*
  391. * Locate the struct page for both the matching buddy in our
  392. * pair (buddy1) and the combined O(n+1) page they form (page).
  393. *
  394. * 1) Any buddy B1 will have an order O twin B2 which satisfies
  395. * the following equation:
  396. * B2 = B1 ^ (1 << O)
  397. * For example, if the starting buddy (buddy2) is #8 its order
  398. * 1 buddy is #10:
  399. * B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10
  400. *
  401. * 2) Any buddy B will have an order O+1 parent P which
  402. * satisfies the following equation:
  403. * P = B & ~(1 << O)
  404. *
  405. * Assumption: *_mem_map is contiguous at least up to MAX_ORDER
  406. */
  407. static inline unsigned long
  408. __find_buddy_index(unsigned long page_idx, unsigned int order)
  409. {
  410. return page_idx ^ (1 << order);
  411. }
  412. /*
  413. * This function checks whether a page is free && is the buddy
  414. * we can do coalesce a page and its buddy if
  415. * (a) the buddy is not in a hole &&
  416. * (b) the buddy is in the buddy system &&
  417. * (c) a page and its buddy have the same order &&
  418. * (d) a page and its buddy are in the same zone.
  419. *
  420. * For recording whether a page is in the buddy system, we set ->_mapcount -2.
  421. * Setting, clearing, and testing _mapcount -2 is serialized by zone->lock.
  422. *
  423. * For recording page's order, we use page_private(page).
  424. */
  425. static inline int page_is_buddy(struct page *page, struct page *buddy,
  426. int order)
  427. {
  428. if (!pfn_valid_within(page_to_pfn(buddy)))
  429. return 0;
  430. if (page_zone_id(page) != page_zone_id(buddy))
  431. return 0;
  432. if (page_is_guard(buddy) && page_order(buddy) == order) {
  433. VM_BUG_ON(page_count(buddy) != 0);
  434. return 1;
  435. }
  436. if (PageBuddy(buddy) && page_order(buddy) == order) {
  437. VM_BUG_ON(page_count(buddy) != 0);
  438. return 1;
  439. }
  440. return 0;
  441. }
  442. /*
  443. * Freeing function for a buddy system allocator.
  444. *
  445. * The concept of a buddy system is to maintain direct-mapped table
  446. * (containing bit values) for memory blocks of various "orders".
  447. * The bottom level table contains the map for the smallest allocatable
  448. * units of memory (here, pages), and each level above it describes
  449. * pairs of units from the levels below, hence, "buddies".
  450. * At a high level, all that happens here is marking the table entry
  451. * at the bottom level available, and propagating the changes upward
  452. * as necessary, plus some accounting needed to play nicely with other
  453. * parts of the VM system.
  454. * At each level, we keep a list of pages, which are heads of continuous
  455. * free pages of length of (1 << order) and marked with _mapcount -2. Page's
  456. * order is recorded in page_private(page) field.
  457. * So when we are allocating or freeing one, we can derive the state of the
  458. * other. That is, if we allocate a small block, and both were
  459. * free, the remainder of the region must be split into blocks.
  460. * If a block is freed, and its buddy is also free, then this
  461. * triggers coalescing into a block of larger size.
  462. *
  463. * -- wli
  464. */
  465. static inline void __free_one_page(struct page *page,
  466. struct zone *zone, unsigned int order,
  467. int migratetype)
  468. {
  469. unsigned long page_idx;
  470. unsigned long combined_idx;
  471. unsigned long uninitialized_var(buddy_idx);
  472. struct page *buddy;
  473. if (unlikely(PageCompound(page)))
  474. if (unlikely(destroy_compound_page(page, order)))
  475. return;
  476. VM_BUG_ON(migratetype == -1);
  477. page_idx = page_to_pfn(page) & ((1 << MAX_ORDER) - 1);
  478. VM_BUG_ON(page_idx & ((1 << order) - 1));
  479. VM_BUG_ON(bad_range(zone, page));
  480. while (order < MAX_ORDER-1) {
  481. buddy_idx = __find_buddy_index(page_idx, order);
  482. buddy = page + (buddy_idx - page_idx);
  483. if (!page_is_buddy(page, buddy, order))
  484. break;
  485. /*
  486. * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
  487. * merge with it and move up one order.
  488. */
  489. if (page_is_guard(buddy)) {
  490. clear_page_guard_flag(buddy);
  491. set_page_private(page, 0);
  492. __mod_zone_freepage_state(zone, 1 << order,
  493. migratetype);
  494. } else {
  495. list_del(&buddy->lru);
  496. zone->free_area[order].nr_free--;
  497. rmv_page_order(buddy);
  498. }
  499. combined_idx = buddy_idx & page_idx;
  500. page = page + (combined_idx - page_idx);
  501. page_idx = combined_idx;
  502. order++;
  503. }
  504. set_page_order(page, order);
  505. /*
  506. * If this is not the largest possible page, check if the buddy
  507. * of the next-highest order is free. If it is, it's possible
  508. * that pages are being freed that will coalesce soon. In case,
  509. * that is happening, add the free page to the tail of the list
  510. * so it's less likely to be used soon and more likely to be merged
  511. * as a higher order page
  512. */
  513. if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {
  514. struct page *higher_page, *higher_buddy;
  515. combined_idx = buddy_idx & page_idx;
  516. higher_page = page + (combined_idx - page_idx);
  517. buddy_idx = __find_buddy_index(combined_idx, order + 1);
  518. higher_buddy = higher_page + (buddy_idx - combined_idx);
  519. if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
  520. list_add_tail(&page->lru,
  521. &zone->free_area[order].free_list[migratetype]);
  522. goto out;
  523. }
  524. }
  525. list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
  526. out:
  527. zone->free_area[order].nr_free++;
  528. }
  529. static inline int free_pages_check(struct page *page)
  530. {
  531. if (unlikely(page_mapcount(page) |
  532. (page->mapping != NULL) |
  533. (atomic_read(&page->_count) != 0) |
  534. (page->flags & PAGE_FLAGS_CHECK_AT_FREE) |
  535. (mem_cgroup_bad_page_check(page)))) {
  536. bad_page(page);
  537. return 1;
  538. }
  539. if (page->flags & PAGE_FLAGS_CHECK_AT_PREP)
  540. page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
  541. return 0;
  542. }
  543. /*
  544. * Frees a number of pages from the PCP lists
  545. * Assumes all pages on list are in same zone, and of same order.
  546. * count is the number of pages to free.
  547. *
  548. * If the zone was previously in an "all pages pinned" state then look to
  549. * see if this freeing clears that state.
  550. *
  551. * And clear the zone's pages_scanned counter, to hold off the "all pages are
  552. * pinned" detection logic.
  553. */
  554. static void free_pcppages_bulk(struct zone *zone, int count,
  555. struct per_cpu_pages *pcp)
  556. {
  557. int migratetype = 0;
  558. int batch_free = 0;
  559. int to_free = count;
  560. spin_lock(&zone->lock);
  561. zone->all_unreclaimable = 0;
  562. zone->pages_scanned = 0;
  563. while (to_free) {
  564. struct page *page;
  565. struct list_head *list;
  566. /*
  567. * Remove pages from lists in a round-robin fashion. A
  568. * batch_free count is maintained that is incremented when an
  569. * empty list is encountered. This is so more pages are freed
  570. * off fuller lists instead of spinning excessively around empty
  571. * lists
  572. */
  573. do {
  574. batch_free++;
  575. if (++migratetype == MIGRATE_PCPTYPES)
  576. migratetype = 0;
  577. list = &pcp->lists[migratetype];
  578. } while (list_empty(list));
  579. /* This is the only non-empty list. Free them all. */
  580. if (batch_free == MIGRATE_PCPTYPES)
  581. batch_free = to_free;
  582. do {
  583. int mt; /* migratetype of the to-be-freed page */
  584. page = list_entry(list->prev, struct page, lru);
  585. /* must delete as __free_one_page list manipulates */
  586. list_del(&page->lru);
  587. mt = get_freepage_migratetype(page);
  588. /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */
  589. __free_one_page(page, zone, 0, mt);
  590. trace_mm_page_pcpu_drain(page, 0, mt);
  591. if (is_migrate_cma(mt))
  592. __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 1);
  593. } while (--to_free && --batch_free && !list_empty(list));
  594. }
  595. __mod_zone_page_state(zone, NR_FREE_PAGES, count);
  596. spin_unlock(&zone->lock);
  597. }
  598. static void free_one_page(struct zone *zone, struct page *page, int order,
  599. int migratetype)
  600. {
  601. spin_lock(&zone->lock);
  602. zone->all_unreclaimable = 0;
  603. zone->pages_scanned = 0;
  604. __free_one_page(page, zone, order, migratetype);
  605. if (unlikely(migratetype != MIGRATE_ISOLATE))
  606. __mod_zone_freepage_state(zone, 1 << order, migratetype);
  607. spin_unlock(&zone->lock);
  608. }
  609. static bool free_pages_prepare(struct page *page, unsigned int order)
  610. {
  611. int i;
  612. int bad = 0;
  613. trace_mm_page_free(page, order);
  614. kmemcheck_free_shadow(page, order);
  615. if (PageAnon(page))
  616. page->mapping = NULL;
  617. for (i = 0; i < (1 << order); i++)
  618. bad += free_pages_check(page + i);
  619. if (bad)
  620. return false;
  621. if (!PageHighMem(page)) {
  622. debug_check_no_locks_freed(page_address(page),PAGE_SIZE<<order);
  623. debug_check_no_obj_freed(page_address(page),
  624. PAGE_SIZE << order);
  625. }
  626. arch_free_page(page, order);
  627. kernel_map_pages(page, 1 << order, 0);
  628. return true;
  629. }
  630. static void __free_pages_ok(struct page *page, unsigned int order)
  631. {
  632. unsigned long flags;
  633. int migratetype;
  634. if (!free_pages_prepare(page, order))
  635. return;
  636. local_irq_save(flags);
  637. __count_vm_events(PGFREE, 1 << order);
  638. migratetype = get_pageblock_migratetype(page);
  639. set_freepage_migratetype(page, migratetype);
  640. free_one_page(page_zone(page), page, order, migratetype);
  641. local_irq_restore(flags);
  642. }
  643. void __meminit __free_pages_bootmem(struct page *page, unsigned int order)
  644. {
  645. unsigned int nr_pages = 1 << order;
  646. unsigned int loop;
  647. prefetchw(page);
  648. for (loop = 0; loop < nr_pages; loop++) {
  649. struct page *p = &page[loop];
  650. if (loop + 1 < nr_pages)
  651. prefetchw(p + 1);
  652. __ClearPageReserved(p);
  653. set_page_count(p, 0);
  654. }
  655. set_page_refcounted(page);
  656. __free_pages(page, order);
  657. }
  658. #ifdef CONFIG_CMA
  659. /* Free whole pageblock and set it's migration type to MIGRATE_CMA. */
  660. void __init init_cma_reserved_pageblock(struct page *page)
  661. {
  662. unsigned i = pageblock_nr_pages;
  663. struct page *p = page;
  664. do {
  665. __ClearPageReserved(p);
  666. set_page_count(p, 0);
  667. } while (++p, --i);
  668. set_page_refcounted(page);
  669. set_pageblock_migratetype(page, MIGRATE_CMA);
  670. __free_pages(page, pageblock_order);
  671. totalram_pages += pageblock_nr_pages;
  672. }
  673. #endif
  674. /*
  675. * The order of subdivision here is critical for the IO subsystem.
  676. * Please do not alter this order without good reasons and regression
  677. * testing. Specifically, as large blocks of memory are subdivided,
  678. * the order in which smaller blocks are delivered depends on the order
  679. * they're subdivided in this function. This is the primary factor
  680. * influencing the order in which pages are delivered to the IO
  681. * subsystem according to empirical testing, and this is also justified
  682. * by considering the behavior of a buddy system containing a single
  683. * large block of memory acted on by a series of small allocations.
  684. * This behavior is a critical factor in sglist merging's success.
  685. *
  686. * -- wli
  687. */
  688. static inline void expand(struct zone *zone, struct page *page,
  689. int low, int high, struct free_area *area,
  690. int migratetype)
  691. {
  692. unsigned long size = 1 << high;
  693. while (high > low) {
  694. area--;
  695. high--;
  696. size >>= 1;
  697. VM_BUG_ON(bad_range(zone, &page[size]));
  698. #ifdef CONFIG_DEBUG_PAGEALLOC
  699. if (high < debug_guardpage_minorder()) {
  700. /*
  701. * Mark as guard pages (or page), that will allow to
  702. * merge back to allocator when buddy will be freed.
  703. * Corresponding page table entries will not be touched,
  704. * pages will stay not present in virtual address space
  705. */
  706. INIT_LIST_HEAD(&page[size].lru);
  707. set_page_guard_flag(&page[size]);
  708. set_page_private(&page[size], high);
  709. /* Guard pages are not available for any usage */
  710. __mod_zone_freepage_state(zone, -(1 << high),
  711. migratetype);
  712. continue;
  713. }
  714. #endif
  715. list_add(&page[size].lru, &area->free_list[migratetype]);
  716. area->nr_free++;
  717. set_page_order(&page[size], high);
  718. }
  719. }
  720. /*
  721. * This page is about to be returned from the page allocator
  722. */
  723. static inline int check_new_page(struct page *page)
  724. {
  725. if (unlikely(page_mapcount(page) |
  726. (page->mapping != NULL) |
  727. (atomic_read(&page->_count) != 0) |
  728. (page->flags & PAGE_FLAGS_CHECK_AT_PREP) |
  729. (mem_cgroup_bad_page_check(page)))) {
  730. bad_page(page);
  731. return 1;
  732. }
  733. return 0;
  734. }
  735. static int prep_new_page(struct page *page, int order, gfp_t gfp_flags)
  736. {
  737. int i;
  738. for (i = 0; i < (1 << order); i++) {
  739. struct page *p = page + i;
  740. if (unlikely(check_new_page(p)))
  741. return 1;
  742. }
  743. set_page_private(page, 0);
  744. set_page_refcounted(page);
  745. arch_alloc_page(page, order);
  746. kernel_map_pages(page, 1 << order, 1);
  747. if (gfp_flags & __GFP_ZERO)
  748. prep_zero_page(page, order, gfp_flags);
  749. if (order && (gfp_flags & __GFP_COMP))
  750. prep_compound_page(page, order);
  751. return 0;
  752. }
  753. /*
  754. * Go through the free lists for the given migratetype and remove
  755. * the smallest available page from the freelists
  756. */
  757. static inline
  758. struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
  759. int migratetype)
  760. {
  761. unsigned int current_order;
  762. struct free_area * area;
  763. struct page *page;
  764. /* Find a page of the appropriate size in the preferred list */
  765. for (current_order = order; current_order < MAX_ORDER; ++current_order) {
  766. area = &(zone->free_area[current_order]);
  767. if (list_empty(&area->free_list[migratetype]))
  768. continue;
  769. page = list_entry(area->free_list[migratetype].next,
  770. struct page, lru);
  771. list_del(&page->lru);
  772. rmv_page_order(page);
  773. area->nr_free--;
  774. expand(zone, page, order, current_order, area, migratetype);
  775. return page;
  776. }
  777. return NULL;
  778. }
  779. /*
  780. * This array describes the order lists are fallen back to when
  781. * the free lists for the desirable migrate type are depleted
  782. */
  783. static int fallbacks[MIGRATE_TYPES][4] = {
  784. [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE },
  785. [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE },
  786. #ifdef CONFIG_CMA
  787. [MIGRATE_MOVABLE] = { MIGRATE_CMA, MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },
  788. [MIGRATE_CMA] = { MIGRATE_RESERVE }, /* Never used */
  789. #else
  790. [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },
  791. #endif
  792. [MIGRATE_RESERVE] = { MIGRATE_RESERVE }, /* Never used */
  793. [MIGRATE_ISOLATE] = { MIGRATE_RESERVE }, /* Never used */
  794. };
  795. /*
  796. * Move the free pages in a range to the free lists of the requested type.
  797. * Note that start_page and end_pages are not aligned on a pageblock
  798. * boundary. If alignment is required, use move_freepages_block()
  799. */
  800. int move_freepages(struct zone *zone,
  801. struct page *start_page, struct page *end_page,
  802. int migratetype)
  803. {
  804. struct page *page;
  805. unsigned long order;
  806. int pages_moved = 0;
  807. #ifndef CONFIG_HOLES_IN_ZONE
  808. /*
  809. * page_zone is not safe to call in this context when
  810. * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant
  811. * anyway as we check zone boundaries in move_freepages_block().
  812. * Remove at a later date when no bug reports exist related to
  813. * grouping pages by mobility
  814. */
  815. BUG_ON(page_zone(start_page) != page_zone(end_page));
  816. #endif
  817. for (page = start_page; page <= end_page;) {
  818. /* Make sure we are not inadvertently changing nodes */
  819. VM_BUG_ON(page_to_nid(page) != zone_to_nid(zone));
  820. if (!pfn_valid_within(page_to_pfn(page))) {
  821. page++;
  822. continue;
  823. }
  824. if (!PageBuddy(page)) {
  825. page++;
  826. continue;
  827. }
  828. order = page_order(page);
  829. list_move(&page->lru,
  830. &zone->free_area[order].free_list[migratetype]);
  831. set_freepage_migratetype(page, migratetype);
  832. page += 1 << order;
  833. pages_moved += 1 << order;
  834. }
  835. return pages_moved;
  836. }
  837. int move_freepages_block(struct zone *zone, struct page *page,
  838. int migratetype)
  839. {
  840. unsigned long start_pfn, end_pfn;
  841. struct page *start_page, *end_page;
  842. start_pfn = page_to_pfn(page);
  843. start_pfn = start_pfn & ~(pageblock_nr_pages-1);
  844. start_page = pfn_to_page(start_pfn);
  845. end_page = start_page + pageblock_nr_pages - 1;
  846. end_pfn = start_pfn + pageblock_nr_pages - 1;
  847. /* Do not cross zone boundaries */
  848. if (start_pfn < zone->zone_start_pfn)
  849. start_page = page;
  850. if (end_pfn >= zone->zone_start_pfn + zone->spanned_pages)
  851. return 0;
  852. return move_freepages(zone, start_page, end_page, migratetype);
  853. }
  854. static void change_pageblock_range(struct page *pageblock_page,
  855. int start_order, int migratetype)
  856. {
  857. int nr_pageblocks = 1 << (start_order - pageblock_order);
  858. while (nr_pageblocks--) {
  859. set_pageblock_migratetype(pageblock_page, migratetype);
  860. pageblock_page += pageblock_nr_pages;
  861. }
  862. }
  863. /* Remove an element from the buddy allocator from the fallback list */
  864. static inline struct page *
  865. __rmqueue_fallback(struct zone *zone, int order, int start_migratetype)
  866. {
  867. struct free_area * area;
  868. int current_order;
  869. struct page *page;
  870. int migratetype, i;
  871. /* Find the largest possible block of pages in the other list */
  872. for (current_order = MAX_ORDER-1; current_order >= order;
  873. --current_order) {
  874. for (i = 0;; i++) {
  875. migratetype = fallbacks[start_migratetype][i];
  876. /* MIGRATE_RESERVE handled later if necessary */
  877. if (migratetype == MIGRATE_RESERVE)
  878. break;
  879. area = &(zone->free_area[current_order]);
  880. if (list_empty(&area->free_list[migratetype]))
  881. continue;
  882. page = list_entry(area->free_list[migratetype].next,
  883. struct page, lru);
  884. area->nr_free--;
  885. /*
  886. * If breaking a large block of pages, move all free
  887. * pages to the preferred allocation list. If falling
  888. * back for a reclaimable kernel allocation, be more
  889. * aggressive about taking ownership of free pages
  890. *
  891. * On the other hand, never change migration
  892. * type of MIGRATE_CMA pageblocks nor move CMA
  893. * pages on different free lists. We don't
  894. * want unmovable pages to be allocated from
  895. * MIGRATE_CMA areas.
  896. */
  897. if (!is_migrate_cma(migratetype) &&
  898. (unlikely(current_order >= pageblock_order / 2) ||
  899. start_migratetype == MIGRATE_RECLAIMABLE ||
  900. page_group_by_mobility_disabled)) {
  901. int pages;
  902. pages = move_freepages_block(zone, page,
  903. start_migratetype);
  904. /* Claim the whole block if over half of it is free */
  905. if (pages >= (1 << (pageblock_order-1)) ||
  906. page_group_by_mobility_disabled)
  907. set_pageblock_migratetype(page,
  908. start_migratetype);
  909. migratetype = start_migratetype;
  910. }
  911. /* Remove the page from the freelists */
  912. list_del(&page->lru);
  913. rmv_page_order(page);
  914. /* Take ownership for orders >= pageblock_order */
  915. if (current_order >= pageblock_order &&
  916. !is_migrate_cma(migratetype))
  917. change_pageblock_range(page, current_order,
  918. start_migratetype);
  919. expand(zone, page, order, current_order, area,
  920. is_migrate_cma(migratetype)
  921. ? migratetype : start_migratetype);
  922. trace_mm_page_alloc_extfrag(page, order, current_order,
  923. start_migratetype, migratetype);
  924. return page;
  925. }
  926. }
  927. return NULL;
  928. }
  929. /*
  930. * Do the hard work of removing an element from the buddy allocator.
  931. * Call me with the zone->lock already held.
  932. */
  933. static struct page *__rmqueue(struct zone *zone, unsigned int order,
  934. int migratetype)
  935. {
  936. struct page *page;
  937. retry_reserve:
  938. page = __rmqueue_smallest(zone, order, migratetype);
  939. if (unlikely(!page) && migratetype != MIGRATE_RESERVE) {
  940. page = __rmqueue_fallback(zone, order, migratetype);
  941. /*
  942. * Use MIGRATE_RESERVE rather than fail an allocation. goto
  943. * is used because __rmqueue_smallest is an inline function
  944. * and we want just one call site
  945. */
  946. if (!page) {
  947. migratetype = MIGRATE_RESERVE;
  948. goto retry_reserve;
  949. }
  950. }
  951. trace_mm_page_alloc_zone_locked(page, order, migratetype);
  952. return page;
  953. }
  954. /*
  955. * Obtain a specified number of elements from the buddy allocator, all under
  956. * a single hold of the lock, for efficiency. Add them to the supplied list.
  957. * Returns the number of new pages which were placed at *list.
  958. */
  959. static int rmqueue_bulk(struct zone *zone, unsigned int order,
  960. unsigned long count, struct list_head *list,
  961. int migratetype, int cold)
  962. {
  963. int mt = migratetype, i;
  964. spin_lock(&zone->lock);
  965. for (i = 0; i < count; ++i) {
  966. struct page *page = __rmqueue(zone, order, migratetype);
  967. if (unlikely(page == NULL))
  968. break;
  969. /*
  970. * Split buddy pages returned by expand() are received here
  971. * in physical page order. The page is added to the callers and
  972. * list and the list head then moves forward. From the callers
  973. * perspective, the linked list is ordered by page number in
  974. * some conditions. This is useful for IO devices that can
  975. * merge IO requests if the physical pages are ordered
  976. * properly.
  977. */
  978. if (likely(cold == 0))
  979. list_add(&page->lru, list);
  980. else
  981. list_add_tail(&page->lru, list);
  982. if (IS_ENABLED(CONFIG_CMA)) {
  983. mt = get_pageblock_migratetype(page);
  984. if (!is_migrate_cma(mt) && mt != MIGRATE_ISOLATE)
  985. mt = migratetype;
  986. }
  987. set_freepage_migratetype(page, mt);
  988. list = &page->lru;
  989. if (is_migrate_cma(mt))
  990. __mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
  991. -(1 << order));
  992. }
  993. __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
  994. spin_unlock(&zone->lock);
  995. return i;
  996. }
  997. #ifdef CONFIG_NUMA
  998. /*
  999. * Called from the vmstat counter updater to drain pagesets of this
  1000. * currently executing processor on remote nodes after they have
  1001. * expired.
  1002. *
  1003. * Note that this function must be called with the thread pinned to
  1004. * a single processor.
  1005. */
  1006. void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
  1007. {
  1008. unsigned long flags;
  1009. int to_drain;
  1010. local_irq_save(flags);
  1011. if (pcp->count >= pcp->batch)
  1012. to_drain = pcp->batch;
  1013. else
  1014. to_drain = pcp->count;
  1015. if (to_drain > 0) {
  1016. free_pcppages_bulk(zone, to_drain, pcp);
  1017. pcp->count -= to_drain;
  1018. }
  1019. local_irq_restore(flags);
  1020. }
  1021. #endif
  1022. /*
  1023. * Drain pages of the indicated processor.
  1024. *
  1025. * The processor must either be the current processor and the
  1026. * thread pinned to the current processor or a processor that
  1027. * is not online.
  1028. */
  1029. static void drain_pages(unsigned int cpu)
  1030. {
  1031. unsigned long flags;
  1032. struct zone *zone;
  1033. for_each_populated_zone(zone) {
  1034. struct per_cpu_pageset *pset;
  1035. struct per_cpu_pages *pcp;
  1036. local_irq_save(flags);
  1037. pset = per_cpu_ptr(zone->pageset, cpu);
  1038. pcp = &pset->pcp;
  1039. if (pcp->count) {
  1040. free_pcppages_bulk(zone, pcp->count, pcp);
  1041. pcp->count = 0;
  1042. }
  1043. local_irq_restore(flags);
  1044. }
  1045. }
  1046. /*
  1047. * Spill all of this CPU's per-cpu pages back into the buddy allocator.
  1048. */
  1049. void drain_local_pages(void *arg)
  1050. {
  1051. drain_pages(smp_processor_id());
  1052. }
  1053. /*
  1054. * Spill all the per-cpu pages from all CPUs back into the buddy allocator.
  1055. *
  1056. * Note that this code is protected against sending an IPI to an offline
  1057. * CPU but does not guarantee sending an IPI to newly hotplugged CPUs:
  1058. * on_each_cpu_mask() blocks hotplug and won't talk to offlined CPUs but
  1059. * nothing keeps CPUs from showing up after we populated the cpumask and
  1060. * before the call to on_each_cpu_mask().
  1061. */
  1062. void drain_all_pages(void)
  1063. {
  1064. int cpu;
  1065. struct per_cpu_pageset *pcp;
  1066. struct zone *zone;
  1067. /*
  1068. * Allocate in the BSS so we wont require allocation in
  1069. * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y
  1070. */
  1071. static cpumask_t cpus_with_pcps;
  1072. /*
  1073. * We don't care about racing with CPU hotplug event
  1074. * as offline notification will cause the notified
  1075. * cpu to drain that CPU pcps and on_each_cpu_mask
  1076. * disables preemption as part of its processing
  1077. */
  1078. for_each_online_cpu(cpu) {
  1079. bool has_pcps = false;
  1080. for_each_populated_zone(zone) {
  1081. pcp = per_cpu_ptr(zone->pageset, cpu);
  1082. if (pcp->pcp.count) {
  1083. has_pcps = true;
  1084. break;
  1085. }
  1086. }
  1087. if (has_pcps)
  1088. cpumask_set_cpu(cpu, &cpus_with_pcps);
  1089. else
  1090. cpumask_clear_cpu(cpu, &cpus_with_pcps);
  1091. }
  1092. on_each_cpu_mask(&cpus_with_pcps, drain_local_pages, NULL, 1);
  1093. }
  1094. #ifdef CONFIG_HIBERNATION
  1095. void mark_free_pages(struct zone *zone)
  1096. {
  1097. unsigned long pfn, max_zone_pfn;
  1098. unsigned long flags;
  1099. int order, t;
  1100. struct list_head *curr;
  1101. if (!zone->spanned_pages)
  1102. return;
  1103. spin_lock_irqsave(&zone->lock, flags);
  1104. max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
  1105. for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
  1106. if (pfn_valid(pfn)) {
  1107. struct page *page = pfn_to_page(pfn);
  1108. if (!swsusp_page_is_forbidden(page))
  1109. swsusp_unset_page_free(page);
  1110. }
  1111. for_each_migratetype_order(order, t) {
  1112. list_for_each(curr, &zone->free_area[order].free_list[t]) {
  1113. unsigned long i;
  1114. pfn = page_to_pfn(list_entry(curr, struct page, lru));
  1115. for (i = 0; i < (1UL << order); i++)
  1116. swsusp_set_page_free(pfn_to_page(pfn + i));
  1117. }
  1118. }
  1119. spin_unlock_irqrestore(&zone->lock, flags);
  1120. }
  1121. #endif /* CONFIG_PM */
  1122. /*
  1123. * Free a 0-order page
  1124. * cold == 1 ? free a cold page : free a hot page
  1125. */
  1126. void free_hot_cold_page(struct page *page, int cold)
  1127. {
  1128. struct zone *zone = page_zone(page);
  1129. struct per_cpu_pages *pcp;
  1130. unsigned long flags;
  1131. int migratetype;
  1132. if (!free_pages_prepare(page, 0))
  1133. return;
  1134. migratetype = get_pageblock_migratetype(page);
  1135. set_freepage_migratetype(page, migratetype);
  1136. local_irq_save(flags);
  1137. __count_vm_event(PGFREE);
  1138. /*
  1139. * We only track unmovable, reclaimable and movable on pcp lists.
  1140. * Free ISOLATE pages back to the allocator because they are being
  1141. * offlined but treat RESERVE as movable pages so we can get those
  1142. * areas back if necessary. Otherwise, we may have to free
  1143. * excessively into the page allocator
  1144. */
  1145. if (migratetype >= MIGRATE_PCPTYPES) {
  1146. if (unlikely(migratetype == MIGRATE_ISOLATE)) {
  1147. free_one_page(zone, page, 0, migratetype);
  1148. goto out;
  1149. }
  1150. migratetype = MIGRATE_MOVABLE;
  1151. }
  1152. pcp = &this_cpu_ptr(zone->pageset)->pcp;
  1153. if (cold)
  1154. list_add_tail(&page->lru, &pcp->lists[migratetype]);
  1155. else
  1156. list_add(&page->lru, &pcp->lists[migratetype]);
  1157. pcp->count++;
  1158. if (pcp->count >= pcp->high) {
  1159. free_pcppages_bulk(zone, pcp->batch, pcp);
  1160. pcp->count -= pcp->batch;
  1161. }
  1162. out:
  1163. local_irq_restore(flags);
  1164. }
  1165. /*
  1166. * Free a list of 0-order pages
  1167. */
  1168. void free_hot_cold_page_list(struct list_head *list, int cold)
  1169. {
  1170. struct page *page, *next;
  1171. list_for_each_entry_safe(page, next, list, lru) {
  1172. trace_mm_page_free_batched(page, cold);
  1173. free_hot_cold_page(page, cold);
  1174. }
  1175. }
  1176. /*
  1177. * split_page takes a non-compound higher-order page, and splits it into
  1178. * n (1<<order) sub-pages: page[0..n]
  1179. * Each sub-page must be freed individually.
  1180. *
  1181. * Note: this is probably too low level an operation for use in drivers.
  1182. * Please consult with lkml before using this in your driver.
  1183. */
  1184. void split_page(struct page *page, unsigned int order)
  1185. {
  1186. int i;
  1187. VM_BUG_ON(PageCompound(page));
  1188. VM_BUG_ON(!page_count(page));
  1189. #ifdef CONFIG_KMEMCHECK
  1190. /*
  1191. * Split shadow pages too, because free(page[0]) would
  1192. * otherwise free the whole shadow.
  1193. */
  1194. if (kmemcheck_page_is_tracked(page))
  1195. split_page(virt_to_page(page[0].shadow), order);
  1196. #endif
  1197. for (i = 1; i < (1 << order); i++)
  1198. set_page_refcounted(page + i);
  1199. }
  1200. /*
  1201. * Similar to the split_page family of functions except that the page
  1202. * required at the given order and being isolated now to prevent races
  1203. * with parallel allocators
  1204. */
  1205. int capture_free_page(struct page *page, int alloc_order, int migratetype)
  1206. {
  1207. unsigned int order;
  1208. unsigned long watermark;
  1209. struct zone *zone;
  1210. int mt;
  1211. BUG_ON(!PageBuddy(page));
  1212. zone = page_zone(page);
  1213. order = page_order(page);
  1214. /* Obey watermarks as if the page was being allocated */
  1215. watermark = low_wmark_pages(zone) + (1 << order);
  1216. if (!zone_watermark_ok(zone, 0, watermark, 0, 0))
  1217. return 0;
  1218. /* Remove page from free list */
  1219. list_del(&page->lru);
  1220. zone->free_area[order].nr_free--;
  1221. rmv_page_order(page);
  1222. mt = get_pageblock_migratetype(page);
  1223. if (unlikely(mt != MIGRATE_ISOLATE))
  1224. __mod_zone_freepage_state(zone, -(1UL << order), mt);
  1225. if (alloc_order != order)
  1226. expand(zone, page, alloc_order, order,
  1227. &zone->free_area[order], migratetype);
  1228. /* Set the pageblock if the captured page is at least a pageblock */
  1229. if (order >= pageblock_order - 1) {
  1230. struct page *endpage = page + (1 << order) - 1;
  1231. for (; page < endpage; page += pageblock_nr_pages) {
  1232. int mt = get_pageblock_migratetype(page);
  1233. if (mt != MIGRATE_ISOLATE && !is_migrate_cma(mt))
  1234. set_pageblock_migratetype(page,
  1235. MIGRATE_MOVABLE);
  1236. }
  1237. }
  1238. return 1UL << order;
  1239. }
  1240. /*
  1241. * Similar to split_page except the page is already free. As this is only
  1242. * being used for migration, the migratetype of the block also changes.
  1243. * As this is called with interrupts disabled, the caller is responsible
  1244. * for calling arch_alloc_page() and kernel_map_page() after interrupts
  1245. * are enabled.
  1246. *
  1247. * Note: this is probably too low level an operation for use in drivers.
  1248. * Please consult with lkml before using this in your driver.
  1249. */
  1250. int split_free_page(struct page *page)
  1251. {
  1252. unsigned int order;
  1253. int nr_pages;
  1254. BUG_ON(!PageBuddy(page));
  1255. order = page_order(page);
  1256. nr_pages = capture_free_page(page, order, 0);
  1257. if (!nr_pages)
  1258. return 0;
  1259. /* Split into individual pages */
  1260. set_page_refcounted(page);
  1261. split_page(page, order);
  1262. return nr_pages;
  1263. }
  1264. /*
  1265. * Really, prep_compound_page() should be called from __rmqueue_bulk(). But
  1266. * we cheat by calling it from here, in the order > 0 path. Saves a branch
  1267. * or two.
  1268. */
  1269. static inline
  1270. struct page *buffered_rmqueue(struct zone *preferred_zone,
  1271. struct zone *zone, int order, gfp_t gfp_flags,
  1272. int migratetype)
  1273. {
  1274. unsigned long flags;
  1275. struct page *page;
  1276. int cold = !!(gfp_flags & __GFP_COLD);
  1277. again:
  1278. if (likely(order == 0)) {
  1279. struct per_cpu_pages *pcp;
  1280. struct list_head *list;
  1281. local_irq_save(flags);
  1282. pcp = &this_cpu_ptr(zone->pageset)->pcp;
  1283. list = &pcp->lists[migratetype];
  1284. if (list_empty(list)) {
  1285. pcp->count += rmqueue_bulk(zone, 0,
  1286. pcp->batch, list,
  1287. migratetype, cold);
  1288. if (unlikely(list_empty(list)))
  1289. goto failed;
  1290. }
  1291. if (cold)
  1292. page = list_entry(list->prev, struct page, lru);
  1293. else
  1294. page = list_entry(list->next, struct page, lru);
  1295. list_del(&page->lru);
  1296. pcp->count--;
  1297. } else {
  1298. if (unlikely(gfp_flags & __GFP_NOFAIL)) {
  1299. /*
  1300. * __GFP_NOFAIL is not to be used in new code.
  1301. *
  1302. * All __GFP_NOFAIL callers should be fixed so that they
  1303. * properly detect and handle allocation failures.
  1304. *
  1305. * We most definitely don't want callers attempting to
  1306. * allocate greater than order-1 page units with
  1307. * __GFP_NOFAIL.
  1308. */
  1309. WARN_ON_ONCE(order > 1);
  1310. }
  1311. spin_lock_irqsave(&zone->lock, flags);
  1312. page = __rmqueue(zone, order, migratetype);
  1313. spin_unlock(&zone->lock);
  1314. if (!page)
  1315. goto failed;
  1316. __mod_zone_freepage_state(zone, -(1 << order),
  1317. get_pageblock_migratetype(page));
  1318. }
  1319. __count_zone_vm_events(PGALLOC, zone, 1 << order);
  1320. zone_statistics(preferred_zone, zone, gfp_flags);
  1321. local_irq_restore(flags);
  1322. VM_BUG_ON(bad_range(zone, page));
  1323. if (prep_new_page(page, order, gfp_flags))
  1324. goto again;
  1325. return page;
  1326. failed:
  1327. local_irq_restore(flags);
  1328. return NULL;
  1329. }
  1330. #ifdef CONFIG_FAIL_PAGE_ALLOC
  1331. static struct {
  1332. struct fault_attr attr;
  1333. u32 ignore_gfp_highmem;
  1334. u32 ignore_gfp_wait;
  1335. u32 min_order;
  1336. } fail_page_alloc = {
  1337. .attr = FAULT_ATTR_INITIALIZER,
  1338. .ignore_gfp_wait = 1,
  1339. .ignore_gfp_highmem = 1,
  1340. .min_order = 1,
  1341. };
  1342. static int __init setup_fail_page_alloc(char *str)
  1343. {
  1344. return setup_fault_attr(&fail_page_alloc.attr, str);
  1345. }
  1346. __setup("fail_page_alloc=", setup_fail_page_alloc);
  1347. static bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
  1348. {
  1349. if (order < fail_page_alloc.min_order)
  1350. return false;
  1351. if (gfp_mask & __GFP_NOFAIL)
  1352. return false;
  1353. if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
  1354. return false;
  1355. if (fail_page_alloc.ignore_gfp_wait && (gfp_mask & __GFP_WAIT))
  1356. return false;
  1357. return should_fail(&fail_page_alloc.attr, 1 << order);
  1358. }
  1359. #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
  1360. static int __init fail_page_alloc_debugfs(void)
  1361. {
  1362. umode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
  1363. struct dentry *dir;
  1364. dir = fault_create_debugfs_attr("fail_page_alloc", NULL,
  1365. &fail_page_alloc.attr);
  1366. if (IS_ERR(dir))
  1367. return PTR_ERR(dir);
  1368. if (!debugfs_create_bool("ignore-gfp-wait", mode, dir,
  1369. &fail_page_alloc.ignore_gfp_wait))
  1370. goto fail;
  1371. if (!debugfs_create_bool("ignore-gfp-highmem", mode, dir,
  1372. &fail_page_alloc.ignore_gfp_highmem))
  1373. goto fail;
  1374. if (!debugfs_create_u32("min-order", mode, dir,
  1375. &fail_page_alloc.min_order))
  1376. goto fail;
  1377. return 0;
  1378. fail:
  1379. debugfs_remove_recursive(dir);
  1380. return -ENOMEM;
  1381. }
  1382. late_initcall(fail_page_alloc_debugfs);
  1383. #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
  1384. #else /* CONFIG_FAIL_PAGE_ALLOC */
  1385. static inline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
  1386. {
  1387. return false;
  1388. }
  1389. #endif /* CONFIG_FAIL_PAGE_ALLOC */
  1390. /*
  1391. * Return true if free pages are above 'mark'. This takes into account the order
  1392. * of the allocation.
  1393. */
  1394. static bool __zone_watermark_ok(struct zone *z, int order, unsigned long mark,
  1395. int classzone_idx, int alloc_flags, long free_pages)
  1396. {
  1397. /* free_pages my go negative - that's OK */
  1398. long min = mark;
  1399. long lowmem_reserve = z->lowmem_reserve[classzone_idx];
  1400. int o;
  1401. free_pages -= (1 << order) - 1;
  1402. if (alloc_flags & ALLOC_HIGH)
  1403. min -= min / 2;
  1404. if (alloc_flags & ALLOC_HARDER)
  1405. min -= min / 4;
  1406. #ifdef CONFIG_CMA
  1407. /* If allocation can't use CMA areas don't use free CMA pages */
  1408. if (!(alloc_flags & ALLOC_CMA))
  1409. free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES);
  1410. #endif
  1411. if (free_pages <= min + lowmem_reserve)
  1412. return false;
  1413. for (o = 0; o < order; o++) {
  1414. /* At the next order, this order's pages become unavailable */
  1415. free_pages -= z->free_area[o].nr_free << o;
  1416. /* Require fewer higher order pages to be free */
  1417. min >>= 1;
  1418. if (free_pages <= min)
  1419. return false;
  1420. }
  1421. return true;
  1422. }
  1423. #ifdef CONFIG_MEMORY_ISOLATION
  1424. static inline unsigned long nr_zone_isolate_freepages(struct zone *zone)
  1425. {
  1426. if (unlikely(zone->nr_pageblock_isolate))
  1427. return zone->nr_pageblock_isolate * pageblock_nr_pages;
  1428. return 0;
  1429. }
  1430. #else
  1431. static inline unsigned long nr_zone_isolate_freepages(struct zone *zone)
  1432. {
  1433. return 0;
  1434. }
  1435. #endif
  1436. bool zone_watermark_ok(struct zone *z, int order, unsigned long mark,
  1437. int classzone_idx, int alloc_flags)
  1438. {
  1439. return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
  1440. zone_page_state(z, NR_FREE_PAGES));
  1441. }
  1442. bool zone_watermark_ok_safe(struct zone *z, int order, unsigned long mark,
  1443. int classzone_idx, int alloc_flags)
  1444. {
  1445. long free_pages = zone_page_state(z, NR_FREE_PAGES);
  1446. if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
  1447. free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
  1448. /*
  1449. * If the zone has MIGRATE_ISOLATE type free pages, we should consider
  1450. * it. nr_zone_isolate_freepages is never accurate so kswapd might not
  1451. * sleep although it could do so. But this is more desirable for memory
  1452. * hotplug than sleeping which can cause a livelock in the direct
  1453. * reclaim path.
  1454. */
  1455. free_pages -= nr_zone_isolate_freepages(z);
  1456. return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
  1457. free_pages);
  1458. }
  1459. #ifdef CONFIG_NUMA
  1460. /*
  1461. * zlc_setup - Setup for "zonelist cache". Uses cached zone data to
  1462. * skip over zones that are not allowed by the cpuset, or that have
  1463. * been recently (in last second) found to be nearly full. See further
  1464. * comments in mmzone.h. Reduces cache footprint of zonelist scans
  1465. * that have to skip over a lot of full or unallowed zones.
  1466. *
  1467. * If the zonelist cache is present in the passed in zonelist, then
  1468. * returns a pointer to the allowed node mask (either the current
  1469. * tasks mems_allowed, or node_states[N_HIGH_MEMORY].)
  1470. *
  1471. * If the zonelist cache is not available for this zonelist, does
  1472. * nothing and returns NULL.
  1473. *
  1474. * If the fullzones BITMAP in the zonelist cache is stale (more than
  1475. * a second since last zap'd) then we zap it out (clear its bits.)
  1476. *
  1477. * We hold off even calling zlc_setup, until after we've checked the
  1478. * first zone in the zonelist, on the theory that most allocations will
  1479. * be satisfied from that first zone, so best to examine that zone as
  1480. * quickly as we can.
  1481. */
  1482. static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
  1483. {
  1484. struct zonelist_cache *zlc; /* cached zonelist speedup info */
  1485. nodemask_t *allowednodes; /* zonelist_cache approximation */
  1486. zlc = zonelist->zlcache_ptr;
  1487. if (!zlc)
  1488. return NULL;
  1489. if (time_after(jiffies, zlc->last_full_zap + HZ)) {
  1490. bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
  1491. zlc->last_full_zap = jiffies;
  1492. }
  1493. allowednodes = !in_interrupt() && (alloc_flags & ALLOC_CPUSET) ?
  1494. &cpuset_current_mems_allowed :
  1495. &node_states[N_HIGH_MEMORY];
  1496. return allowednodes;
  1497. }
  1498. /*
  1499. * Given 'z' scanning a zonelist, run a couple of quick checks to see
  1500. * if it is worth looking at further for free memory:
  1501. * 1) Check that the zone isn't thought to be full (doesn't have its
  1502. * bit set in the zonelist_cache fullzones BITMAP).
  1503. * 2) Check that the zones node (obtained from the zonelist_cache
  1504. * z_to_n[] mapping) is allowed in the passed in allowednodes mask.
  1505. * Return true (non-zero) if zone is worth looking at further, or
  1506. * else return false (zero) if it is not.
  1507. *
  1508. * This check -ignores- the distinction between various watermarks,
  1509. * such as GFP_HIGH, GFP_ATOMIC, PF_MEMALLOC, ... If a zone is
  1510. * found to be full for any variation of these watermarks, it will
  1511. * be considered full for up to one second by all requests, unless
  1512. * we are so low on memory on all allowed nodes that we are forced
  1513. * into the second scan of the zonelist.
  1514. *
  1515. * In the second scan we ignore this zonelist cache and exactly
  1516. * apply the watermarks to all zones, even it is slower to do so.
  1517. * We are low on memory in the second scan, and should leave no stone
  1518. * unturned looking for a free page.
  1519. */
  1520. static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z,
  1521. nodemask_t *allowednodes)
  1522. {
  1523. struct zonelist_cache *zlc; /* cached zonelist speedup info */
  1524. int i; /* index of *z in zonelist zones */
  1525. int n; /* node that zone *z is on */
  1526. zlc = zonelist->zlcache_ptr;
  1527. if (!zlc)
  1528. return 1;
  1529. i = z - zonelist->_zonerefs;
  1530. n = zlc->z_to_n[i];
  1531. /* This zone is worth trying if it is allowed but not full */
  1532. return node_isset(n, *allowednodes) && !test_bit(i, zlc->fullzones);
  1533. }
  1534. /*
  1535. * Given 'z' scanning a zonelist, set the corresponding bit in
  1536. * zlc->fullzones, so that subsequent attempts to allocate a page
  1537. * from that zone don't waste time re-examining it.
  1538. */
  1539. static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z)
  1540. {
  1541. struct zonelist_cache *zlc; /* cached zonelist speedup info */
  1542. int i; /* index of *z in zonelist zones */
  1543. zlc = zonelist->zlcache_ptr;
  1544. if (!zlc)
  1545. return;
  1546. i = z - zonelist->_zonerefs;
  1547. set_bit(i, zlc->fullzones);
  1548. }
  1549. /*
  1550. * clear all zones full, called after direct reclaim makes progress so that
  1551. * a zone that was recently full is not skipped over for up to a second
  1552. */
  1553. static void zlc_clear_zones_full(struct zonelist *zonelist)
  1554. {
  1555. struct zonelist_cache *zlc; /* cached zonelist speedup info */
  1556. zlc = zonelist->zlcache_ptr;
  1557. if (!zlc)
  1558. return;
  1559. bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
  1560. }
  1561. #else /* CONFIG_NUMA */
  1562. static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
  1563. {
  1564. return NULL;
  1565. }
  1566. static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z,
  1567. nodemask_t *allowednodes)
  1568. {
  1569. return 1;
  1570. }
  1571. static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z)
  1572. {
  1573. }
  1574. static void zlc_clear_zones_full(struct zonelist *zonelist)
  1575. {
  1576. }
  1577. #endif /* CONFIG_NUMA */
  1578. /*
  1579. * get_page_from_freelist goes through the zonelist trying to allocate
  1580. * a page.
  1581. */
  1582. static struct page *
  1583. get_page_from_freelist(gfp_t gfp_mask, nodemask_t *nodemask, unsigned int order,
  1584. struct zonelist *zonelist, int high_zoneidx, int alloc_flags,
  1585. struct zone *preferred_zone, int migratetype)
  1586. {
  1587. struct zoneref *z;
  1588. struct page *page = NULL;
  1589. int classzone_idx;
  1590. struct zone *zone;
  1591. nodemask_t *allowednodes = NULL;/* zonelist_cache approximation */
  1592. int zlc_active = 0; /* set if using zonelist_cache */
  1593. int did_zlc_setup = 0; /* just call zlc_setup() one time */
  1594. classzone_idx = zone_idx(preferred_zone);
  1595. zonelist_scan:
  1596. /*
  1597. * Scan zonelist, looking for a zone with enough free.
  1598. * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
  1599. */
  1600. for_each_zone_zonelist_nodemask(zone, z, zonelist,
  1601. high_zoneidx, nodemask) {
  1602. if (NUMA_BUILD && zlc_active &&
  1603. !zlc_zone_worth_trying(zonelist, z, allowednodes))
  1604. continue;
  1605. if ((alloc_flags & ALLOC_CPUSET) &&
  1606. !cpuset_zone_allowed_softwall(zone, gfp_mask))
  1607. continue;
  1608. /*
  1609. * When allocating a page cache page for writing, we
  1610. * want to get it from a zone that is within its dirty
  1611. * limit, such that no single zone holds more than its
  1612. * proportional share of globally allowed dirty pages.
  1613. * The dirty limits take into account the zone's
  1614. * lowmem reserves and high watermark so that kswapd
  1615. * should be able to balance it without having to
  1616. * write pages from its LRU list.
  1617. *
  1618. * This may look like it could increase pressure on
  1619. * lower zones by failing allocations in higher zones
  1620. * before they are full. But the pages that do spill
  1621. * over are limited as the lower zones are protected
  1622. * by this very same mechanism. It should not become
  1623. * a practical burden to them.
  1624. *
  1625. * XXX: For now, allow allocations to potentially
  1626. * exceed the per-zone dirty limit in the slowpath
  1627. * (ALLOC_WMARK_LOW unset) before going into reclaim,
  1628. * which is important when on a NUMA setup the allowed
  1629. * zones are together not big enough to reach the
  1630. * global limit. The proper fix for these situations
  1631. * will require awareness of zones in the
  1632. * dirty-throttling and the flusher threads.
  1633. */
  1634. if ((alloc_flags & ALLOC_WMARK_LOW) &&
  1635. (gfp_mask & __GFP_WRITE) && !zone_dirty_ok(zone))
  1636. goto this_zone_full;
  1637. BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
  1638. if (!(alloc_flags & ALLOC_NO_WATERMARKS)) {
  1639. unsigned long mark;
  1640. int ret;
  1641. mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK];
  1642. if (zone_watermark_ok(zone, order, mark,
  1643. classzone_idx, alloc_flags))
  1644. goto try_this_zone;
  1645. if (NUMA_BUILD && !did_zlc_setup && nr_online_nodes > 1) {
  1646. /*
  1647. * we do zlc_setup if there are multiple nodes
  1648. * and before considering the first zone allowed
  1649. * by the cpuset.
  1650. */
  1651. allowednodes = zlc_setup(zonelist, alloc_flags);
  1652. zlc_active = 1;
  1653. did_zlc_setup = 1;
  1654. }
  1655. if (zone_reclaim_mode == 0)
  1656. goto this_zone_full;
  1657. /*
  1658. * As we may have just activated ZLC, check if the first
  1659. * eligible zone has failed zone_reclaim recently.
  1660. */
  1661. if (NUMA_BUILD && zlc_active &&
  1662. !zlc_zone_worth_trying(zonelist, z, allowednodes))
  1663. continue;
  1664. ret = zone_reclaim(zone, gfp_mask, order);
  1665. switch (ret) {
  1666. case ZONE_RECLAIM_NOSCAN:
  1667. /* did not scan */
  1668. continue;
  1669. case ZONE_RECLAIM_FULL:
  1670. /* scanned but unreclaimable */
  1671. continue;
  1672. default:
  1673. /* did we reclaim enough */
  1674. if (!zone_watermark_ok(zone, order, mark,
  1675. classzone_idx, alloc_flags))
  1676. goto this_zone_full;
  1677. }
  1678. }
  1679. try_this_zone:
  1680. page = buffered_rmqueue(preferred_zone, zone, order,
  1681. gfp_mask, migratetype);
  1682. if (page)
  1683. break;
  1684. this_zone_full:
  1685. if (NUMA_BUILD)
  1686. zlc_mark_zone_full(zonelist, z);
  1687. }
  1688. if (unlikely(NUMA_BUILD && page == NULL && zlc_active)) {
  1689. /* Disable zlc cache for second zonelist scan */
  1690. zlc_active = 0;
  1691. goto zonelist_scan;
  1692. }
  1693. if (page)
  1694. /*
  1695. * page->pfmemalloc is set when ALLOC_NO_WATERMARKS was
  1696. * necessary to allocate the page. The expectation is
  1697. * that the caller is taking steps that will free more
  1698. * memory. The caller should avoid the page being used
  1699. * for !PFMEMALLOC purposes.
  1700. */
  1701. page->pfmemalloc = !!(alloc_flags & ALLOC_NO_WATERMARKS);
  1702. return page;
  1703. }
  1704. /*
  1705. * Large machines with many possible nodes should not always dump per-node
  1706. * meminfo in irq context.
  1707. */
  1708. static inline bool should_suppress_show_mem(void)
  1709. {
  1710. bool ret = false;
  1711. #if NODES_SHIFT > 8
  1712. ret = in_interrupt();
  1713. #endif
  1714. return ret;
  1715. }
  1716. static DEFINE_RATELIMIT_STATE(nopage_rs,
  1717. DEFAULT_RATELIMIT_INTERVAL,
  1718. DEFAULT_RATELIMIT_BURST);
  1719. void warn_alloc_failed(gfp_t gfp_mask, int order, const char *fmt, ...)
  1720. {
  1721. unsigned int filter = SHOW_MEM_FILTER_NODES;
  1722. if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) ||
  1723. debug_guardpage_minorder() > 0)
  1724. return;
  1725. /*
  1726. * This documents exceptions given to allocations in certain
  1727. * contexts that are allowed to allocate outside current's set
  1728. * of allowed nodes.
  1729. */
  1730. if (!(gfp_mask & __GFP_NOMEMALLOC))
  1731. if (test_thread_flag(TIF_MEMDIE) ||
  1732. (current->flags & (PF_MEMALLOC | PF_EXITING)))
  1733. filter &= ~SHOW_MEM_FILTER_NODES;
  1734. if (in_interrupt() || !(gfp_mask & __GFP_WAIT))
  1735. filter &= ~SHOW_MEM_FILTER_NODES;
  1736. if (fmt) {
  1737. struct va_format vaf;
  1738. va_list args;
  1739. va_start(args, fmt);
  1740. vaf.fmt = fmt;
  1741. vaf.va = &args;
  1742. pr_warn("%pV", &vaf);
  1743. va_end(args);
  1744. }
  1745. pr_warn("%s: page allocation failure: order:%d, mode:0x%x\n",
  1746. current->comm, order, gfp_mask);
  1747. dump_stack();
  1748. if (!should_suppress_show_mem())
  1749. show_mem(filter);
  1750. }
  1751. static inline int
  1752. should_alloc_retry(gfp_t gfp_mask, unsigned int order,
  1753. unsigned long did_some_progress,
  1754. unsigned long pages_reclaimed)
  1755. {
  1756. /* Do not loop if specifically requested */
  1757. if (gfp_mask & __GFP_NORETRY)
  1758. return 0;
  1759. /* Always retry if specifically requested */
  1760. if (gfp_mask & __GFP_NOFAIL)
  1761. return 1;
  1762. /*
  1763. * Suspend converts GFP_KERNEL to __GFP_WAIT which can prevent reclaim
  1764. * making forward progress without invoking OOM. Suspend also disables
  1765. * storage devices so kswapd will not help. Bail if we are suspending.
  1766. */
  1767. if (!did_some_progress && pm_suspended_storage())
  1768. return 0;
  1769. /*
  1770. * In this implementation, order <= PAGE_ALLOC_COSTLY_ORDER
  1771. * means __GFP_NOFAIL, but that may not be true in other
  1772. * implementations.
  1773. */
  1774. if (order <= PAGE_ALLOC_COSTLY_ORDER)
  1775. return 1;
  1776. /*
  1777. * For order > PAGE_ALLOC_COSTLY_ORDER, if __GFP_REPEAT is
  1778. * specified, then we retry until we no longer reclaim any pages
  1779. * (above), or we've reclaimed an order of pages at least as
  1780. * large as the allocation's order. In both cases, if the
  1781. * allocation still fails, we stop retrying.
  1782. */
  1783. if (gfp_mask & __GFP_REPEAT && pages_reclaimed < (1 << order))
  1784. return 1;
  1785. return 0;
  1786. }
  1787. static inline struct page *
  1788. __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
  1789. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1790. nodemask_t *nodemask, struct zone *preferred_zone,
  1791. int migratetype)
  1792. {
  1793. struct page *page;
  1794. /* Acquire the OOM killer lock for the zones in zonelist */
  1795. if (!try_set_zonelist_oom(zonelist, gfp_mask)) {
  1796. schedule_timeout_uninterruptible(1);
  1797. return NULL;
  1798. }
  1799. /*
  1800. * Go through the zonelist yet one more time, keep very high watermark
  1801. * here, this is only to catch a parallel oom killing, we must fail if
  1802. * we're still under heavy pressure.
  1803. */
  1804. page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask,
  1805. order, zonelist, high_zoneidx,
  1806. ALLOC_WMARK_HIGH|ALLOC_CPUSET,
  1807. preferred_zone, migratetype);
  1808. if (page)
  1809. goto out;
  1810. if (!(gfp_mask & __GFP_NOFAIL)) {
  1811. /* The OOM killer will not help higher order allocs */
  1812. if (order > PAGE_ALLOC_COSTLY_ORDER)
  1813. goto out;
  1814. /* The OOM killer does not needlessly kill tasks for lowmem */
  1815. if (high_zoneidx < ZONE_NORMAL)
  1816. goto out;
  1817. /*
  1818. * GFP_THISNODE contains __GFP_NORETRY and we never hit this.
  1819. * Sanity check for bare calls of __GFP_THISNODE, not real OOM.
  1820. * The caller should handle page allocation failure by itself if
  1821. * it specifies __GFP_THISNODE.
  1822. * Note: Hugepage uses it but will hit PAGE_ALLOC_COSTLY_ORDER.
  1823. */
  1824. if (gfp_mask & __GFP_THISNODE)
  1825. goto out;
  1826. }
  1827. /* Exhausted what can be done so it's blamo time */
  1828. out_of_memory(zonelist, gfp_mask, order, nodemask, false);
  1829. out:
  1830. clear_zonelist_oom(zonelist, gfp_mask);
  1831. return page;
  1832. }
  1833. #ifdef CONFIG_COMPACTION
  1834. /* Try memory compaction for high-order allocations before reclaim */
  1835. static struct page *
  1836. __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
  1837. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1838. nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
  1839. int migratetype, bool sync_migration,
  1840. bool *contended_compaction, bool *deferred_compaction,
  1841. unsigned long *did_some_progress)
  1842. {
  1843. struct page *page = NULL;
  1844. if (!order)
  1845. return NULL;
  1846. if (compaction_deferred(preferred_zone, order)) {
  1847. *deferred_compaction = true;
  1848. return NULL;
  1849. }
  1850. current->flags |= PF_MEMALLOC;
  1851. *did_some_progress = try_to_compact_pages(zonelist, order, gfp_mask,
  1852. nodemask, sync_migration,
  1853. contended_compaction, &page);
  1854. current->flags &= ~PF_MEMALLOC;
  1855. /* If compaction captured a page, prep and use it */
  1856. if (page) {
  1857. prep_new_page(page, order, gfp_mask);
  1858. goto got_page;
  1859. }
  1860. if (*did_some_progress != COMPACT_SKIPPED) {
  1861. /* Page migration frees to the PCP lists but we want merging */
  1862. drain_pages(get_cpu());
  1863. put_cpu();
  1864. page = get_page_from_freelist(gfp_mask, nodemask,
  1865. order, zonelist, high_zoneidx,
  1866. alloc_flags & ~ALLOC_NO_WATERMARKS,
  1867. preferred_zone, migratetype);
  1868. if (page) {
  1869. got_page:
  1870. preferred_zone->compact_blockskip_flush = false;
  1871. preferred_zone->compact_considered = 0;
  1872. preferred_zone->compact_defer_shift = 0;
  1873. if (order >= preferred_zone->compact_order_failed)
  1874. preferred_zone->compact_order_failed = order + 1;
  1875. count_vm_event(COMPACTSUCCESS);
  1876. return page;
  1877. }
  1878. /*
  1879. * It's bad if compaction run occurs and fails.
  1880. * The most likely reason is that pages exist,
  1881. * but not enough to satisfy watermarks.
  1882. */
  1883. count_vm_event(COMPACTFAIL);
  1884. /*
  1885. * As async compaction considers a subset of pageblocks, only
  1886. * defer if the failure was a sync compaction failure.
  1887. */
  1888. if (sync_migration)
  1889. defer_compaction(preferred_zone, order);
  1890. cond_resched();
  1891. }
  1892. return NULL;
  1893. }
  1894. #else
  1895. static inline struct page *
  1896. __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
  1897. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1898. nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
  1899. int migratetype, bool sync_migration,
  1900. bool *contended_compaction, bool *deferred_compaction,
  1901. unsigned long *did_some_progress)
  1902. {
  1903. return NULL;
  1904. }
  1905. #endif /* CONFIG_COMPACTION */
  1906. /* Perform direct synchronous page reclaim */
  1907. static int
  1908. __perform_reclaim(gfp_t gfp_mask, unsigned int order, struct zonelist *zonelist,
  1909. nodemask_t *nodemask)
  1910. {
  1911. struct reclaim_state reclaim_state;
  1912. int progress;
  1913. cond_resched();
  1914. /* We now go into synchronous reclaim */
  1915. cpuset_memory_pressure_bump();
  1916. current->flags |= PF_MEMALLOC;
  1917. lockdep_set_current_reclaim_state(gfp_mask);
  1918. reclaim_state.reclaimed_slab = 0;
  1919. current->reclaim_state = &reclaim_state;
  1920. progress = try_to_free_pages(zonelist, order, gfp_mask, nodemask);
  1921. current->reclaim_state = NULL;
  1922. lockdep_clear_current_reclaim_state();
  1923. current->flags &= ~PF_MEMALLOC;
  1924. cond_resched();
  1925. return progress;
  1926. }
  1927. /* The really slow allocator path where we enter direct reclaim */
  1928. static inline struct page *
  1929. __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
  1930. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1931. nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
  1932. int migratetype, unsigned long *did_some_progress)
  1933. {
  1934. struct page *page = NULL;
  1935. bool drained = false;
  1936. *did_some_progress = __perform_reclaim(gfp_mask, order, zonelist,
  1937. nodemask);
  1938. if (unlikely(!(*did_some_progress)))
  1939. return NULL;
  1940. /* After successful reclaim, reconsider all zones for allocation */
  1941. if (NUMA_BUILD)
  1942. zlc_clear_zones_full(zonelist);
  1943. retry:
  1944. page = get_page_from_freelist(gfp_mask, nodemask, order,
  1945. zonelist, high_zoneidx,
  1946. alloc_flags & ~ALLOC_NO_WATERMARKS,
  1947. preferred_zone, migratetype);
  1948. /*
  1949. * If an allocation failed after direct reclaim, it could be because
  1950. * pages are pinned on the per-cpu lists. Drain them and try again
  1951. */
  1952. if (!page && !drained) {
  1953. drain_all_pages();
  1954. drained = true;
  1955. goto retry;
  1956. }
  1957. return page;
  1958. }
  1959. /*
  1960. * This is called in the allocator slow-path if the allocation request is of
  1961. * sufficient urgency to ignore watermarks and take other desperate measures
  1962. */
  1963. static inline struct page *
  1964. __alloc_pages_high_priority(gfp_t gfp_mask, unsigned int order,
  1965. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1966. nodemask_t *nodemask, struct zone *preferred_zone,
  1967. int migratetype)
  1968. {
  1969. struct page *page;
  1970. do {
  1971. page = get_page_from_freelist(gfp_mask, nodemask, order,
  1972. zonelist, high_zoneidx, ALLOC_NO_WATERMARKS,
  1973. preferred_zone, migratetype);
  1974. if (!page && gfp_mask & __GFP_NOFAIL)
  1975. wait_iff_congested(preferred_zone, BLK_RW_ASYNC, HZ/50);
  1976. } while (!page && (gfp_mask & __GFP_NOFAIL));
  1977. return page;
  1978. }
  1979. static inline
  1980. void wake_all_kswapd(unsigned int order, struct zonelist *zonelist,
  1981. enum zone_type high_zoneidx,
  1982. enum zone_type classzone_idx)
  1983. {
  1984. struct zoneref *z;
  1985. struct zone *zone;
  1986. for_each_zone_zonelist(zone, z, zonelist, high_zoneidx)
  1987. wakeup_kswapd(zone, order, classzone_idx);
  1988. }
  1989. static inline int
  1990. gfp_to_alloc_flags(gfp_t gfp_mask)
  1991. {
  1992. int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
  1993. const gfp_t wait = gfp_mask & __GFP_WAIT;
  1994. /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */
  1995. BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);
  1996. /*
  1997. * The caller may dip into page reserves a bit more if the caller
  1998. * cannot run direct reclaim, or if the caller has realtime scheduling
  1999. * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will
  2000. * set both ALLOC_HARDER (!wait) and ALLOC_HIGH (__GFP_HIGH).
  2001. */
  2002. alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH);
  2003. if (!wait) {
  2004. /*
  2005. * Not worth trying to allocate harder for
  2006. * __GFP_NOMEMALLOC even if it can't schedule.
  2007. */
  2008. if (!(gfp_mask & __GFP_NOMEMALLOC))
  2009. alloc_flags |= ALLOC_HARDER;
  2010. /*
  2011. * Ignore cpuset if GFP_ATOMIC (!wait) rather than fail alloc.
  2012. * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
  2013. */
  2014. alloc_flags &= ~ALLOC_CPUSET;
  2015. } else if (unlikely(rt_task(current)) && !in_interrupt())
  2016. alloc_flags |= ALLOC_HARDER;
  2017. if (likely(!(gfp_mask & __GFP_NOMEMALLOC))) {
  2018. if (gfp_mask & __GFP_MEMALLOC)
  2019. alloc_flags |= ALLOC_NO_WATERMARKS;
  2020. else if (in_serving_softirq() && (current->flags & PF_MEMALLOC))
  2021. alloc_flags |= ALLOC_NO_WATERMARKS;
  2022. else if (!in_interrupt() &&
  2023. ((current->flags & PF_MEMALLOC) ||
  2024. unlikely(test_thread_flag(TIF_MEMDIE))))
  2025. alloc_flags |= ALLOC_NO_WATERMARKS;
  2026. }
  2027. #ifdef CONFIG_CMA
  2028. if (allocflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
  2029. alloc_flags |= ALLOC_CMA;
  2030. #endif
  2031. return alloc_flags;
  2032. }
  2033. bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
  2034. {
  2035. return !!(gfp_to_alloc_flags(gfp_mask) & ALLOC_NO_WATERMARKS);
  2036. }
  2037. static inline struct page *
  2038. __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
  2039. struct zonelist *zonelist, enum zone_type high_zoneidx,
  2040. nodemask_t *nodemask, struct zone *preferred_zone,
  2041. int migratetype)
  2042. {
  2043. const gfp_t wait = gfp_mask & __GFP_WAIT;
  2044. struct page *page = NULL;
  2045. int alloc_flags;
  2046. unsigned long pages_reclaimed = 0;
  2047. unsigned long did_some_progress;
  2048. bool sync_migration = false;
  2049. bool deferred_compaction = false;
  2050. bool contended_compaction = false;
  2051. /*
  2052. * In the slowpath, we sanity check order to avoid ever trying to
  2053. * reclaim >= MAX_ORDER areas which will never succeed. Callers may
  2054. * be using allocators in order of preference for an area that is
  2055. * too large.
  2056. */
  2057. if (order >= MAX_ORDER) {
  2058. WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));
  2059. return NULL;
  2060. }
  2061. /*
  2062. * GFP_THISNODE (meaning __GFP_THISNODE, __GFP_NORETRY and
  2063. * __GFP_NOWARN set) should not cause reclaim since the subsystem
  2064. * (f.e. slab) using GFP_THISNODE may choose to trigger reclaim
  2065. * using a larger set of nodes after it has established that the
  2066. * allowed per node queues are empty and that nodes are
  2067. * over allocated.
  2068. */
  2069. if (NUMA_BUILD && (gfp_mask & GFP_THISNODE) == GFP_THISNODE)
  2070. goto nopage;
  2071. restart:
  2072. wake_all_kswapd(order, zonelist, high_zoneidx,
  2073. zone_idx(preferred_zone));
  2074. /*
  2075. * OK, we're below the kswapd watermark and have kicked background
  2076. * reclaim. Now things get more complex, so set up alloc_flags according
  2077. * to how we want to proceed.
  2078. */
  2079. alloc_flags = gfp_to_alloc_flags(gfp_mask);
  2080. /*
  2081. * Find the true preferred zone if the allocation is unconstrained by
  2082. * cpusets.
  2083. */
  2084. if (!(alloc_flags & ALLOC_CPUSET) && !nodemask)
  2085. first_zones_zonelist(zonelist, high_zoneidx, NULL,
  2086. &preferred_zone);
  2087. rebalance:
  2088. /* This is the last chance, in general, before the goto nopage. */
  2089. page = get_page_from_freelist(gfp_mask, nodemask, order, zonelist,
  2090. high_zoneidx, alloc_flags & ~ALLOC_NO_WATERMARKS,
  2091. preferred_zone, migratetype);
  2092. if (page)
  2093. goto got_pg;
  2094. /* Allocate without watermarks if the context allows */
  2095. if (alloc_flags & ALLOC_NO_WATERMARKS) {
  2096. /*
  2097. * Ignore mempolicies if ALLOC_NO_WATERMARKS on the grounds
  2098. * the allocation is high priority and these type of
  2099. * allocations are system rather than user orientated
  2100. */
  2101. zonelist = node_zonelist(numa_node_id(), gfp_mask);
  2102. page = __alloc_pages_high_priority(gfp_mask, order,
  2103. zonelist, high_zoneidx, nodemask,
  2104. preferred_zone, migratetype);
  2105. if (page) {
  2106. goto got_pg;
  2107. }
  2108. }
  2109. /* Atomic allocations - we can't balance anything */
  2110. if (!wait)
  2111. goto nopage;
  2112. /* Avoid recursion of direct reclaim */
  2113. if (current->flags & PF_MEMALLOC)
  2114. goto nopage;
  2115. /* Avoid allocations with no watermarks from looping endlessly */
  2116. if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL))
  2117. goto nopage;
  2118. /*
  2119. * Try direct compaction. The first pass is asynchronous. Subsequent
  2120. * attempts after direct reclaim are synchronous
  2121. */
  2122. page = __alloc_pages_direct_compact(gfp_mask, order,
  2123. zonelist, high_zoneidx,
  2124. nodemask,
  2125. alloc_flags, preferred_zone,
  2126. migratetype, sync_migration,
  2127. &contended_compaction,
  2128. &deferred_compaction,
  2129. &did_some_progress);
  2130. if (page)
  2131. goto got_pg;
  2132. sync_migration = true;
  2133. /*
  2134. * If compaction is deferred for high-order allocations, it is because
  2135. * sync compaction recently failed. In this is the case and the caller
  2136. * requested a movable allocation that does not heavily disrupt the
  2137. * system then fail the allocation instead of entering direct reclaim.
  2138. */
  2139. if ((deferred_compaction || contended_compaction) &&
  2140. (gfp_mask & (__GFP_MOVABLE|__GFP_REPEAT)) == __GFP_MOVABLE)
  2141. goto nopage;
  2142. /* Try direct reclaim and then allocating */
  2143. page = __alloc_pages_direct_reclaim(gfp_mask, order,
  2144. zonelist, high_zoneidx,
  2145. nodemask,
  2146. alloc_flags, preferred_zone,
  2147. migratetype, &did_some_progress);
  2148. if (page)
  2149. goto got_pg;
  2150. /*
  2151. * If we failed to make any progress reclaiming, then we are
  2152. * running out of options and have to consider going OOM
  2153. */
  2154. if (!did_some_progress) {
  2155. if ((gfp_mask & __GFP_FS) && !(gfp_mask & __GFP_NORETRY)) {
  2156. if (oom_killer_disabled)
  2157. goto nopage;
  2158. /* Coredumps can quickly deplete all memory reserves */
  2159. if ((current->flags & PF_DUMPCORE) &&
  2160. !(gfp_mask & __GFP_NOFAIL))
  2161. goto nopage;
  2162. page = __alloc_pages_may_oom(gfp_mask, order,
  2163. zonelist, high_zoneidx,
  2164. nodemask, preferred_zone,
  2165. migratetype);
  2166. if (page)
  2167. goto got_pg;
  2168. if (!(gfp_mask & __GFP_NOFAIL)) {
  2169. /*
  2170. * The oom killer is not called for high-order
  2171. * allocations that may fail, so if no progress
  2172. * is being made, there are no other options and
  2173. * retrying is unlikely to help.
  2174. */
  2175. if (order > PAGE_ALLOC_COSTLY_ORDER)
  2176. goto nopage;
  2177. /*
  2178. * The oom killer is not called for lowmem
  2179. * allocations to prevent needlessly killing
  2180. * innocent tasks.
  2181. */
  2182. if (high_zoneidx < ZONE_NORMAL)
  2183. goto nopage;
  2184. }
  2185. goto restart;
  2186. }
  2187. }
  2188. /* Check if we should retry the allocation */
  2189. pages_reclaimed += did_some_progress;
  2190. if (should_alloc_retry(gfp_mask, order, did_some_progress,
  2191. pages_reclaimed)) {
  2192. /* Wait for some write requests to complete then retry */
  2193. wait_iff_congested(preferred_zone, BLK_RW_ASYNC, HZ/50);
  2194. goto rebalance;
  2195. } else {
  2196. /*
  2197. * High-order allocations do not necessarily loop after
  2198. * direct reclaim and reclaim/compaction depends on compaction
  2199. * being called after reclaim so call directly if necessary
  2200. */
  2201. page = __alloc_pages_direct_compact(gfp_mask, order,
  2202. zonelist, high_zoneidx,
  2203. nodemask,
  2204. alloc_flags, preferred_zone,
  2205. migratetype, sync_migration,
  2206. &contended_compaction,
  2207. &deferred_compaction,
  2208. &did_some_progress);
  2209. if (page)
  2210. goto got_pg;
  2211. }
  2212. nopage:
  2213. warn_alloc_failed(gfp_mask, order, NULL);
  2214. return page;
  2215. got_pg:
  2216. if (kmemcheck_enabled)
  2217. kmemcheck_pagealloc_alloc(page, order, gfp_mask);
  2218. return page;
  2219. }
  2220. /*
  2221. * This is the 'heart' of the zoned buddy allocator.
  2222. */
  2223. struct page *
  2224. __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
  2225. struct zonelist *zonelist, nodemask_t *nodemask)
  2226. {
  2227. enum zone_type high_zoneidx = gfp_zone(gfp_mask);
  2228. struct zone *preferred_zone;
  2229. struct page *page = NULL;
  2230. int migratetype = allocflags_to_migratetype(gfp_mask);
  2231. unsigned int cpuset_mems_cookie;
  2232. int alloc_flags = ALLOC_WMARK_LOW|ALLOC_CPUSET;
  2233. gfp_mask &= gfp_allowed_mask;
  2234. lockdep_trace_alloc(gfp_mask);
  2235. might_sleep_if(gfp_mask & __GFP_WAIT);
  2236. if (should_fail_alloc_page(gfp_mask, order))
  2237. return NULL;
  2238. /*
  2239. * Check the zones suitable for the gfp_mask contain at least one
  2240. * valid zone. It's possible to have an empty zonelist as a result
  2241. * of GFP_THISNODE and a memoryless node
  2242. */
  2243. if (unlikely(!zonelist->_zonerefs->zone))
  2244. return NULL;
  2245. retry_cpuset:
  2246. cpuset_mems_cookie = get_mems_allowed();
  2247. /* The preferred zone is used for statistics later */
  2248. first_zones_zonelist(zonelist, high_zoneidx,
  2249. nodemask ? : &cpuset_current_mems_allowed,
  2250. &preferred_zone);
  2251. if (!preferred_zone)
  2252. goto out;
  2253. #ifdef CONFIG_CMA
  2254. if (allocflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
  2255. alloc_flags |= ALLOC_CMA;
  2256. #endif
  2257. /* First allocation attempt */
  2258. page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask, order,
  2259. zonelist, high_zoneidx, alloc_flags,
  2260. preferred_zone, migratetype);
  2261. if (unlikely(!page))
  2262. page = __alloc_pages_slowpath(gfp_mask, order,
  2263. zonelist, high_zoneidx, nodemask,
  2264. preferred_zone, migratetype);
  2265. trace_mm_page_alloc(page, order, gfp_mask, migratetype);
  2266. out:
  2267. /*
  2268. * When updating a task's mems_allowed, it is possible to race with
  2269. * parallel threads in such a way that an allocation can fail while
  2270. * the mask is being updated. If a page allocation is about to fail,
  2271. * check if the cpuset changed during allocation and if so, retry.
  2272. */
  2273. if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
  2274. goto retry_cpuset;
  2275. return page;
  2276. }
  2277. EXPORT_SYMBOL(__alloc_pages_nodemask);
  2278. /*
  2279. * Common helper functions.
  2280. */
  2281. unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
  2282. {
  2283. struct page *page;
  2284. /*
  2285. * __get_free_pages() returns a 32-bit address, which cannot represent
  2286. * a highmem page
  2287. */
  2288. VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0);
  2289. page = alloc_pages(gfp_mask, order);
  2290. if (!page)
  2291. return 0;
  2292. return (unsigned long) page_address(page);
  2293. }
  2294. EXPORT_SYMBOL(__get_free_pages);
  2295. unsigned long get_zeroed_page(gfp_t gfp_mask)
  2296. {
  2297. return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
  2298. }
  2299. EXPORT_SYMBOL(get_zeroed_page);
  2300. void __free_pages(struct page *page, unsigned int order)
  2301. {
  2302. if (put_page_testzero(page)) {
  2303. if (order == 0)
  2304. free_hot_cold_page(page, 0);
  2305. else
  2306. __free_pages_ok(page, order);
  2307. }
  2308. }
  2309. EXPORT_SYMBOL(__free_pages);
  2310. void free_pages(unsigned long addr, unsigned int order)
  2311. {
  2312. if (addr != 0) {
  2313. VM_BUG_ON(!virt_addr_valid((void *)addr));
  2314. __free_pages(virt_to_page((void *)addr), order);
  2315. }
  2316. }
  2317. EXPORT_SYMBOL(free_pages);
  2318. static void *make_alloc_exact(unsigned long addr, unsigned order, size_t size)
  2319. {
  2320. if (addr) {
  2321. unsigned long alloc_end = addr + (PAGE_SIZE << order);
  2322. unsigned long used = addr + PAGE_ALIGN(size);
  2323. split_page(virt_to_page((void *)addr), order);
  2324. while (used < alloc_end) {
  2325. free_page(used);
  2326. used += PAGE_SIZE;
  2327. }
  2328. }
  2329. return (void *)addr;
  2330. }
  2331. /**
  2332. * alloc_pages_exact - allocate an exact number physically-contiguous pages.
  2333. * @size: the number of bytes to allocate
  2334. * @gfp_mask: GFP flags for the allocation
  2335. *
  2336. * This function is similar to alloc_pages(), except that it allocates the
  2337. * minimum number of pages to satisfy the request. alloc_pages() can only
  2338. * allocate memory in power-of-two pages.
  2339. *
  2340. * This function is also limited by MAX_ORDER.
  2341. *
  2342. * Memory allocated by this function must be released by free_pages_exact().
  2343. */
  2344. void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
  2345. {
  2346. unsigned int order = get_order(size);
  2347. unsigned long addr;
  2348. addr = __get_free_pages(gfp_mask, order);
  2349. return make_alloc_exact(addr, order, size);
  2350. }
  2351. EXPORT_SYMBOL(alloc_pages_exact);
  2352. /**
  2353. * alloc_pages_exact_nid - allocate an exact number of physically-contiguous
  2354. * pages on a node.
  2355. * @nid: the preferred node ID where memory should be allocated
  2356. * @size: the number of bytes to allocate
  2357. * @gfp_mask: GFP flags for the allocation
  2358. *
  2359. * Like alloc_pages_exact(), but try to allocate on node nid first before falling
  2360. * back.
  2361. * Note this is not alloc_pages_exact_node() which allocates on a specific node,
  2362. * but is not exact.
  2363. */
  2364. void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask)
  2365. {
  2366. unsigned order = get_order(size);
  2367. struct page *p = alloc_pages_node(nid, gfp_mask, order);
  2368. if (!p)
  2369. return NULL;
  2370. return make_alloc_exact((unsigned long)page_address(p), order, size);
  2371. }
  2372. EXPORT_SYMBOL(alloc_pages_exact_nid);
  2373. /**
  2374. * free_pages_exact - release memory allocated via alloc_pages_exact()
  2375. * @virt: the value returned by alloc_pages_exact.
  2376. * @size: size of allocation, same value as passed to alloc_pages_exact().
  2377. *
  2378. * Release the memory allocated by a previous call to alloc_pages_exact.
  2379. */
  2380. void free_pages_exact(void *virt, size_t size)
  2381. {
  2382. unsigned long addr = (unsigned long)virt;
  2383. unsigned long end = addr + PAGE_ALIGN(size);
  2384. while (addr < end) {
  2385. free_page(addr);
  2386. addr += PAGE_SIZE;
  2387. }
  2388. }
  2389. EXPORT_SYMBOL(free_pages_exact);
  2390. static unsigned int nr_free_zone_pages(int offset)
  2391. {
  2392. struct zoneref *z;
  2393. struct zone *zone;
  2394. /* Just pick one node, since fallback list is circular */
  2395. unsigned int sum = 0;
  2396. struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
  2397. for_each_zone_zonelist(zone, z, zonelist, offset) {
  2398. unsigned long size = zone->present_pages;
  2399. unsigned long high = high_wmark_pages(zone);
  2400. if (size > high)
  2401. sum += size - high;
  2402. }
  2403. return sum;
  2404. }
  2405. /*
  2406. * Amount of free RAM allocatable within ZONE_DMA and ZONE_NORMAL
  2407. */
  2408. unsigned int nr_free_buffer_pages(void)
  2409. {
  2410. return nr_free_zone_pages(gfp_zone(GFP_USER));
  2411. }
  2412. EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
  2413. /*
  2414. * Amount of free RAM allocatable within all zones
  2415. */
  2416. unsigned int nr_free_pagecache_pages(void)
  2417. {
  2418. return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
  2419. }
  2420. static inline void show_node(struct zone *zone)
  2421. {
  2422. if (NUMA_BUILD)
  2423. printk("Node %d ", zone_to_nid(zone));
  2424. }
  2425. void si_meminfo(struct sysinfo *val)
  2426. {
  2427. val->totalram = totalram_pages;
  2428. val->sharedram = 0;
  2429. val->freeram = global_page_state(NR_FREE_PAGES);
  2430. val->bufferram = nr_blockdev_pages();
  2431. val->totalhigh = totalhigh_pages;
  2432. val->freehigh = nr_free_highpages();
  2433. val->mem_unit = PAGE_SIZE;
  2434. }
  2435. EXPORT_SYMBOL(si_meminfo);
  2436. #ifdef CONFIG_NUMA
  2437. void si_meminfo_node(struct sysinfo *val, int nid)
  2438. {
  2439. pg_data_t *pgdat = NODE_DATA(nid);
  2440. val->totalram = pgdat->node_present_pages;
  2441. val->freeram = node_page_state(nid, NR_FREE_PAGES);
  2442. #ifdef CONFIG_HIGHMEM
  2443. val->totalhigh = pgdat->node_zones[ZONE_HIGHMEM].present_pages;
  2444. val->freehigh = zone_page_state(&pgdat->node_zones[ZONE_HIGHMEM],
  2445. NR_FREE_PAGES);
  2446. #else
  2447. val->totalhigh = 0;
  2448. val->freehigh = 0;
  2449. #endif
  2450. val->mem_unit = PAGE_SIZE;
  2451. }
  2452. #endif
  2453. /*
  2454. * Determine whether the node should be displayed or not, depending on whether
  2455. * SHOW_MEM_FILTER_NODES was passed to show_free_areas().
  2456. */
  2457. bool skip_free_areas_node(unsigned int flags, int nid)
  2458. {
  2459. bool ret = false;
  2460. unsigned int cpuset_mems_cookie;
  2461. if (!(flags & SHOW_MEM_FILTER_NODES))
  2462. goto out;
  2463. do {
  2464. cpuset_mems_cookie = get_mems_allowed();
  2465. ret = !node_isset(nid, cpuset_current_mems_allowed);
  2466. } while (!put_mems_allowed(cpuset_mems_cookie));
  2467. out:
  2468. return ret;
  2469. }
  2470. #define K(x) ((x) << (PAGE_SHIFT-10))
  2471. /*
  2472. * Show free area list (used inside shift_scroll-lock stuff)
  2473. * We also calculate the percentage fragmentation. We do this by counting the
  2474. * memory on each free list with the exception of the first item on the list.
  2475. * Suppresses nodes that are not allowed by current's cpuset if
  2476. * SHOW_MEM_FILTER_NODES is passed.
  2477. */
  2478. void show_free_areas(unsigned int filter)
  2479. {
  2480. int cpu;
  2481. struct zone *zone;
  2482. for_each_populated_zone(zone) {
  2483. if (skip_free_areas_node(filter, zone_to_nid(zone)))
  2484. continue;
  2485. show_node(zone);
  2486. printk("%s per-cpu:\n", zone->name);
  2487. for_each_online_cpu(cpu) {
  2488. struct per_cpu_pageset *pageset;
  2489. pageset = per_cpu_ptr(zone->pageset, cpu);
  2490. printk("CPU %4d: hi:%5d, btch:%4d usd:%4d\n",
  2491. cpu, pageset->pcp.high,
  2492. pageset->pcp.batch, pageset->pcp.count);
  2493. }
  2494. }
  2495. printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n"
  2496. " active_file:%lu inactive_file:%lu isolated_file:%lu\n"
  2497. " unevictable:%lu"
  2498. " dirty:%lu writeback:%lu unstable:%lu\n"
  2499. " free:%lu slab_reclaimable:%lu slab_unreclaimable:%lu\n"
  2500. " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n"
  2501. " free_cma:%lu\n",
  2502. global_page_state(NR_ACTIVE_ANON),
  2503. global_page_state(NR_INACTIVE_ANON),
  2504. global_page_state(NR_ISOLATED_ANON),
  2505. global_page_state(NR_ACTIVE_FILE),
  2506. global_page_state(NR_INACTIVE_FILE),
  2507. global_page_state(NR_ISOLATED_FILE),
  2508. global_page_state(NR_UNEVICTABLE),
  2509. global_page_state(NR_FILE_DIRTY),
  2510. global_page_state(NR_WRITEBACK),
  2511. global_page_state(NR_UNSTABLE_NFS),
  2512. global_page_state(NR_FREE_PAGES),
  2513. global_page_state(NR_SLAB_RECLAIMABLE),
  2514. global_page_state(NR_SLAB_UNRECLAIMABLE),
  2515. global_page_state(NR_FILE_MAPPED),
  2516. global_page_state(NR_SHMEM),
  2517. global_page_state(NR_PAGETABLE),
  2518. global_page_state(NR_BOUNCE),
  2519. global_page_state(NR_FREE_CMA_PAGES));
  2520. for_each_populated_zone(zone) {
  2521. int i;
  2522. if (skip_free_areas_node(filter, zone_to_nid(zone)))
  2523. continue;
  2524. show_node(zone);
  2525. printk("%s"
  2526. " free:%lukB"
  2527. " min:%lukB"
  2528. " low:%lukB"
  2529. " high:%lukB"
  2530. " active_anon:%lukB"
  2531. " inactive_anon:%lukB"
  2532. " active_file:%lukB"
  2533. " inactive_file:%lukB"
  2534. " unevictable:%lukB"
  2535. " isolated(anon):%lukB"
  2536. " isolated(file):%lukB"
  2537. " present:%lukB"
  2538. " mlocked:%lukB"
  2539. " dirty:%lukB"
  2540. " writeback:%lukB"
  2541. " mapped:%lukB"
  2542. " shmem:%lukB"
  2543. " slab_reclaimable:%lukB"
  2544. " slab_unreclaimable:%lukB"
  2545. " kernel_stack:%lukB"
  2546. " pagetables:%lukB"
  2547. " unstable:%lukB"
  2548. " bounce:%lukB"
  2549. " free_cma:%lukB"
  2550. " writeback_tmp:%lukB"
  2551. " pages_scanned:%lu"
  2552. " all_unreclaimable? %s"
  2553. "\n",
  2554. zone->name,
  2555. K(zone_page_state(zone, NR_FREE_PAGES)),
  2556. K(min_wmark_pages(zone)),
  2557. K(low_wmark_pages(zone)),
  2558. K(high_wmark_pages(zone)),
  2559. K(zone_page_state(zone, NR_ACTIVE_ANON)),
  2560. K(zone_page_state(zone, NR_INACTIVE_ANON)),
  2561. K(zone_page_state(zone, NR_ACTIVE_FILE)),
  2562. K(zone_page_state(zone, NR_INACTIVE_FILE)),
  2563. K(zone_page_state(zone, NR_UNEVICTABLE)),
  2564. K(zone_page_state(zone, NR_ISOLATED_ANON)),
  2565. K(zone_page_state(zone, NR_ISOLATED_FILE)),
  2566. K(zone->present_pages),
  2567. K(zone_page_state(zone, NR_MLOCK)),
  2568. K(zone_page_state(zone, NR_FILE_DIRTY)),
  2569. K(zone_page_state(zone, NR_WRITEBACK)),
  2570. K(zone_page_state(zone, NR_FILE_MAPPED)),
  2571. K(zone_page_state(zone, NR_SHMEM)),
  2572. K(zone_page_state(zone, NR_SLAB_RECLAIMABLE)),
  2573. K(zone_page_state(zone, NR_SLAB_UNRECLAIMABLE)),
  2574. zone_page_state(zone, NR_KERNEL_STACK) *
  2575. THREAD_SIZE / 1024,
  2576. K(zone_page_state(zone, NR_PAGETABLE)),
  2577. K(zone_page_state(zone, NR_UNSTABLE_NFS)),
  2578. K(zone_page_state(zone, NR_BOUNCE)),
  2579. K(zone_page_state(zone, NR_FREE_CMA_PAGES)),
  2580. K(zone_page_state(zone, NR_WRITEBACK_TEMP)),
  2581. zone->pages_scanned,
  2582. (zone->all_unreclaimable ? "yes" : "no")
  2583. );
  2584. printk("lowmem_reserve[]:");
  2585. for (i = 0; i < MAX_NR_ZONES; i++)
  2586. printk(" %lu", zone->lowmem_reserve[i]);
  2587. printk("\n");
  2588. }
  2589. for_each_populated_zone(zone) {
  2590. unsigned long nr[MAX_ORDER], flags, order, total = 0;
  2591. if (skip_free_areas_node(filter, zone_to_nid(zone)))
  2592. continue;
  2593. show_node(zone);
  2594. printk("%s: ", zone->name);
  2595. spin_lock_irqsave(&zone->lock, flags);
  2596. for (order = 0; order < MAX_ORDER; order++) {
  2597. nr[order] = zone->free_area[order].nr_free;
  2598. total += nr[order] << order;
  2599. }
  2600. spin_unlock_irqrestore(&zone->lock, flags);
  2601. for (order = 0; order < MAX_ORDER; order++)
  2602. printk("%lu*%lukB ", nr[order], K(1UL) << order);
  2603. printk("= %lukB\n", K(total));
  2604. }
  2605. printk("%ld total pagecache pages\n", global_page_state(NR_FILE_PAGES));
  2606. show_swap_cache_info();
  2607. }
  2608. static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
  2609. {
  2610. zoneref->zone = zone;
  2611. zoneref->zone_idx = zone_idx(zone);
  2612. }
  2613. /*
  2614. * Builds allocation fallback zone lists.
  2615. *
  2616. * Add all populated zones of a node to the zonelist.
  2617. */
  2618. static int build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist,
  2619. int nr_zones, enum zone_type zone_type)
  2620. {
  2621. struct zone *zone;
  2622. BUG_ON(zone_type >= MAX_NR_ZONES);
  2623. zone_type++;
  2624. do {
  2625. zone_type--;
  2626. zone = pgdat->node_zones + zone_type;
  2627. if (populated_zone(zone)) {
  2628. zoneref_set_zone(zone,
  2629. &zonelist->_zonerefs[nr_zones++]);
  2630. check_highest_zone(zone_type);
  2631. }
  2632. } while (zone_type);
  2633. return nr_zones;
  2634. }
  2635. /*
  2636. * zonelist_order:
  2637. * 0 = automatic detection of better ordering.
  2638. * 1 = order by ([node] distance, -zonetype)
  2639. * 2 = order by (-zonetype, [node] distance)
  2640. *
  2641. * If not NUMA, ZONELIST_ORDER_ZONE and ZONELIST_ORDER_NODE will create
  2642. * the same zonelist. So only NUMA can configure this param.
  2643. */
  2644. #define ZONELIST_ORDER_DEFAULT 0
  2645. #define ZONELIST_ORDER_NODE 1
  2646. #define ZONELIST_ORDER_ZONE 2
  2647. /* zonelist order in the kernel.
  2648. * set_zonelist_order() will set this to NODE or ZONE.
  2649. */
  2650. static int current_zonelist_order = ZONELIST_ORDER_DEFAULT;
  2651. static char zonelist_order_name[3][8] = {"Default", "Node", "Zone"};
  2652. #ifdef CONFIG_NUMA
  2653. /* The value user specified ....changed by config */
  2654. static int user_zonelist_order = ZONELIST_ORDER_DEFAULT;
  2655. /* string for sysctl */
  2656. #define NUMA_ZONELIST_ORDER_LEN 16
  2657. char numa_zonelist_order[16] = "default";
  2658. /*
  2659. * interface for configure zonelist ordering.
  2660. * command line option "numa_zonelist_order"
  2661. * = "[dD]efault - default, automatic configuration.
  2662. * = "[nN]ode - order by node locality, then by zone within node
  2663. * = "[zZ]one - order by zone, then by locality within zone
  2664. */
  2665. static int __parse_numa_zonelist_order(char *s)
  2666. {
  2667. if (*s == 'd' || *s == 'D') {
  2668. user_zonelist_order = ZONELIST_ORDER_DEFAULT;
  2669. } else if (*s == 'n' || *s == 'N') {
  2670. user_zonelist_order = ZONELIST_ORDER_NODE;
  2671. } else if (*s == 'z' || *s == 'Z') {
  2672. user_zonelist_order = ZONELIST_ORDER_ZONE;
  2673. } else {
  2674. printk(KERN_WARNING
  2675. "Ignoring invalid numa_zonelist_order value: "
  2676. "%s\n", s);
  2677. return -EINVAL;
  2678. }
  2679. return 0;
  2680. }
  2681. static __init int setup_numa_zonelist_order(char *s)
  2682. {
  2683. int ret;
  2684. if (!s)
  2685. return 0;
  2686. ret = __parse_numa_zonelist_order(s);
  2687. if (ret == 0)
  2688. strlcpy(numa_zonelist_order, s, NUMA_ZONELIST_ORDER_LEN);
  2689. return ret;
  2690. }
  2691. early_param("numa_zonelist_order", setup_numa_zonelist_order);
  2692. /*
  2693. * sysctl handler for numa_zonelist_order
  2694. */
  2695. int numa_zonelist_order_handler(ctl_table *table, int write,
  2696. void __user *buffer, size_t *length,
  2697. loff_t *ppos)
  2698. {
  2699. char saved_string[NUMA_ZONELIST_ORDER_LEN];
  2700. int ret;
  2701. static DEFINE_MUTEX(zl_order_mutex);
  2702. mutex_lock(&zl_order_mutex);
  2703. if (write)
  2704. strcpy(saved_string, (char*)table->data);
  2705. ret = proc_dostring(table, write, buffer, length, ppos);
  2706. if (ret)
  2707. goto out;
  2708. if (write) {
  2709. int oldval = user_zonelist_order;
  2710. if (__parse_numa_zonelist_order((char*)table->data)) {
  2711. /*
  2712. * bogus value. restore saved string
  2713. */
  2714. strncpy((char*)table->data, saved_string,
  2715. NUMA_ZONELIST_ORDER_LEN);
  2716. user_zonelist_order = oldval;
  2717. } else if (oldval != user_zonelist_order) {
  2718. mutex_lock(&zonelists_mutex);
  2719. build_all_zonelists(NULL, NULL);
  2720. mutex_unlock(&zonelists_mutex);
  2721. }
  2722. }
  2723. out:
  2724. mutex_unlock(&zl_order_mutex);
  2725. return ret;
  2726. }
  2727. #define MAX_NODE_LOAD (nr_online_nodes)
  2728. static int node_load[MAX_NUMNODES];
  2729. /**
  2730. * find_next_best_node - find the next node that should appear in a given node's fallback list
  2731. * @node: node whose fallback list we're appending
  2732. * @used_node_mask: nodemask_t of already used nodes
  2733. *
  2734. * We use a number of factors to determine which is the next node that should
  2735. * appear on a given node's fallback list. The node should not have appeared
  2736. * already in @node's fallback list, and it should be the next closest node
  2737. * according to the distance array (which contains arbitrary distance values
  2738. * from each node to each node in the system), and should also prefer nodes
  2739. * with no CPUs, since presumably they'll have very little allocation pressure
  2740. * on them otherwise.
  2741. * It returns -1 if no node is found.
  2742. */
  2743. static int find_next_best_node(int node, nodemask_t *used_node_mask)
  2744. {
  2745. int n, val;
  2746. int min_val = INT_MAX;
  2747. int best_node = -1;
  2748. const struct cpumask *tmp = cpumask_of_node(0);
  2749. /* Use the local node if we haven't already */
  2750. if (!node_isset(node, *used_node_mask)) {
  2751. node_set(node, *used_node_mask);
  2752. return node;
  2753. }
  2754. for_each_node_state(n, N_HIGH_MEMORY) {
  2755. /* Don't want a node to appear more than once */
  2756. if (node_isset(n, *used_node_mask))
  2757. continue;
  2758. /* Use the distance array to find the distance */
  2759. val = node_distance(node, n);
  2760. /* Penalize nodes under us ("prefer the next node") */
  2761. val += (n < node);
  2762. /* Give preference to headless and unused nodes */
  2763. tmp = cpumask_of_node(n);
  2764. if (!cpumask_empty(tmp))
  2765. val += PENALTY_FOR_NODE_WITH_CPUS;
  2766. /* Slight preference for less loaded node */
  2767. val *= (MAX_NODE_LOAD*MAX_NUMNODES);
  2768. val += node_load[n];
  2769. if (val < min_val) {
  2770. min_val = val;
  2771. best_node = n;
  2772. }
  2773. }
  2774. if (best_node >= 0)
  2775. node_set(best_node, *used_node_mask);
  2776. return best_node;
  2777. }
  2778. /*
  2779. * Build zonelists ordered by node and zones within node.
  2780. * This results in maximum locality--normal zone overflows into local
  2781. * DMA zone, if any--but risks exhausting DMA zone.
  2782. */
  2783. static void build_zonelists_in_node_order(pg_data_t *pgdat, int node)
  2784. {
  2785. int j;
  2786. struct zonelist *zonelist;
  2787. zonelist = &pgdat->node_zonelists[0];
  2788. for (j = 0; zonelist->_zonerefs[j].zone != NULL; j++)
  2789. ;
  2790. j = build_zonelists_node(NODE_DATA(node), zonelist, j,
  2791. MAX_NR_ZONES - 1);
  2792. zonelist->_zonerefs[j].zone = NULL;
  2793. zonelist->_zonerefs[j].zone_idx = 0;
  2794. }
  2795. /*
  2796. * Build gfp_thisnode zonelists
  2797. */
  2798. static void build_thisnode_zonelists(pg_data_t *pgdat)
  2799. {
  2800. int j;
  2801. struct zonelist *zonelist;
  2802. zonelist = &pgdat->node_zonelists[1];
  2803. j = build_zonelists_node(pgdat, zonelist, 0, MAX_NR_ZONES - 1);
  2804. zonelist->_zonerefs[j].zone = NULL;
  2805. zonelist->_zonerefs[j].zone_idx = 0;
  2806. }
  2807. /*
  2808. * Build zonelists ordered by zone and nodes within zones.
  2809. * This results in conserving DMA zone[s] until all Normal memory is
  2810. * exhausted, but results in overflowing to remote node while memory
  2811. * may still exist in local DMA zone.
  2812. */
  2813. static int node_order[MAX_NUMNODES];
  2814. static void build_zonelists_in_zone_order(pg_data_t *pgdat, int nr_nodes)
  2815. {
  2816. int pos, j, node;
  2817. int zone_type; /* needs to be signed */
  2818. struct zone *z;
  2819. struct zonelist *zonelist;
  2820. zonelist = &pgdat->node_zonelists[0];
  2821. pos = 0;
  2822. for (zone_type = MAX_NR_ZONES - 1; zone_type >= 0; zone_type--) {
  2823. for (j = 0; j < nr_nodes; j++) {
  2824. node = node_order[j];
  2825. z = &NODE_DATA(node)->node_zones[zone_type];
  2826. if (populated_zone(z)) {
  2827. zoneref_set_zone(z,
  2828. &zonelist->_zonerefs[pos++]);
  2829. check_highest_zone(zone_type);
  2830. }
  2831. }
  2832. }
  2833. zonelist->_zonerefs[pos].zone = NULL;
  2834. zonelist->_zonerefs[pos].zone_idx = 0;
  2835. }
  2836. static int default_zonelist_order(void)
  2837. {
  2838. int nid, zone_type;
  2839. unsigned long low_kmem_size,total_size;
  2840. struct zone *z;
  2841. int average_size;
  2842. /*
  2843. * ZONE_DMA and ZONE_DMA32 can be very small area in the system.
  2844. * If they are really small and used heavily, the system can fall
  2845. * into OOM very easily.
  2846. * This function detect ZONE_DMA/DMA32 size and configures zone order.
  2847. */
  2848. /* Is there ZONE_NORMAL ? (ex. ppc has only DMA zone..) */
  2849. low_kmem_size = 0;
  2850. total_size = 0;
  2851. for_each_online_node(nid) {
  2852. for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
  2853. z = &NODE_DATA(nid)->node_zones[zone_type];
  2854. if (populated_zone(z)) {
  2855. if (zone_type < ZONE_NORMAL)
  2856. low_kmem_size += z->present_pages;
  2857. total_size += z->present_pages;
  2858. } else if (zone_type == ZONE_NORMAL) {
  2859. /*
  2860. * If any node has only lowmem, then node order
  2861. * is preferred to allow kernel allocations
  2862. * locally; otherwise, they can easily infringe
  2863. * on other nodes when there is an abundance of
  2864. * lowmem available to allocate from.
  2865. */
  2866. return ZONELIST_ORDER_NODE;
  2867. }
  2868. }
  2869. }
  2870. if (!low_kmem_size || /* there are no DMA area. */
  2871. low_kmem_size > total_size/2) /* DMA/DMA32 is big. */
  2872. return ZONELIST_ORDER_NODE;
  2873. /*
  2874. * look into each node's config.
  2875. * If there is a node whose DMA/DMA32 memory is very big area on
  2876. * local memory, NODE_ORDER may be suitable.
  2877. */
  2878. average_size = total_size /
  2879. (nodes_weight(node_states[N_HIGH_MEMORY]) + 1);
  2880. for_each_online_node(nid) {
  2881. low_kmem_size = 0;
  2882. total_size = 0;
  2883. for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
  2884. z = &NODE_DATA(nid)->node_zones[zone_type];
  2885. if (populated_zone(z)) {
  2886. if (zone_type < ZONE_NORMAL)
  2887. low_kmem_size += z->present_pages;
  2888. total_size += z->present_pages;
  2889. }
  2890. }
  2891. if (low_kmem_size &&
  2892. total_size > average_size && /* ignore small node */
  2893. low_kmem_size > total_size * 70/100)
  2894. return ZONELIST_ORDER_NODE;
  2895. }
  2896. return ZONELIST_ORDER_ZONE;
  2897. }
  2898. static void set_zonelist_order(void)
  2899. {
  2900. if (user_zonelist_order == ZONELIST_ORDER_DEFAULT)
  2901. current_zonelist_order = default_zonelist_order();
  2902. else
  2903. current_zonelist_order = user_zonelist_order;
  2904. }
  2905. static void build_zonelists(pg_data_t *pgdat)
  2906. {
  2907. int j, node, load;
  2908. enum zone_type i;
  2909. nodemask_t used_mask;
  2910. int local_node, prev_node;
  2911. struct zonelist *zonelist;
  2912. int order = current_zonelist_order;
  2913. /* initialize zonelists */
  2914. for (i = 0; i < MAX_ZONELISTS; i++) {
  2915. zonelist = pgdat->node_zonelists + i;
  2916. zonelist->_zonerefs[0].zone = NULL;
  2917. zonelist->_zonerefs[0].zone_idx = 0;
  2918. }
  2919. /* NUMA-aware ordering of nodes */
  2920. local_node = pgdat->node_id;
  2921. load = nr_online_nodes;
  2922. prev_node = local_node;
  2923. nodes_clear(used_mask);
  2924. memset(node_order, 0, sizeof(node_order));
  2925. j = 0;
  2926. while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
  2927. int distance = node_distance(local_node, node);
  2928. /*
  2929. * If another node is sufficiently far away then it is better
  2930. * to reclaim pages in a zone before going off node.
  2931. */
  2932. if (distance > RECLAIM_DISTANCE)
  2933. zone_reclaim_mode = 1;
  2934. /*
  2935. * We don't want to pressure a particular node.
  2936. * So adding penalty to the first node in same
  2937. * distance group to make it round-robin.
  2938. */
  2939. if (distance != node_distance(local_node, prev_node))
  2940. node_load[node] = load;
  2941. prev_node = node;
  2942. load--;
  2943. if (order == ZONELIST_ORDER_NODE)
  2944. build_zonelists_in_node_order(pgdat, node);
  2945. else
  2946. node_order[j++] = node; /* remember order */
  2947. }
  2948. if (order == ZONELIST_ORDER_ZONE) {
  2949. /* calculate node order -- i.e., DMA last! */
  2950. build_zonelists_in_zone_order(pgdat, j);
  2951. }
  2952. build_thisnode_zonelists(pgdat);
  2953. }
  2954. /* Construct the zonelist performance cache - see further mmzone.h */
  2955. static void build_zonelist_cache(pg_data_t *pgdat)
  2956. {
  2957. struct zonelist *zonelist;
  2958. struct zonelist_cache *zlc;
  2959. struct zoneref *z;
  2960. zonelist = &pgdat->node_zonelists[0];
  2961. zonelist->zlcache_ptr = zlc = &zonelist->zlcache;
  2962. bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
  2963. for (z = zonelist->_zonerefs; z->zone; z++)
  2964. zlc->z_to_n[z - zonelist->_zonerefs] = zonelist_node_idx(z);
  2965. }
  2966. #ifdef CONFIG_HAVE_MEMORYLESS_NODES
  2967. /*
  2968. * Return node id of node used for "local" allocations.
  2969. * I.e., first node id of first zone in arg node's generic zonelist.
  2970. * Used for initializing percpu 'numa_mem', which is used primarily
  2971. * for kernel allocations, so use GFP_KERNEL flags to locate zonelist.
  2972. */
  2973. int local_memory_node(int node)
  2974. {
  2975. struct zone *zone;
  2976. (void)first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
  2977. gfp_zone(GFP_KERNEL),
  2978. NULL,
  2979. &zone);
  2980. return zone->node;
  2981. }
  2982. #endif
  2983. #else /* CONFIG_NUMA */
  2984. static void set_zonelist_order(void)
  2985. {
  2986. current_zonelist_order = ZONELIST_ORDER_ZONE;
  2987. }
  2988. static void build_zonelists(pg_data_t *pgdat)
  2989. {
  2990. int node, local_node;
  2991. enum zone_type j;
  2992. struct zonelist *zonelist;
  2993. local_node = pgdat->node_id;
  2994. zonelist = &pgdat->node_zonelists[0];
  2995. j = build_zonelists_node(pgdat, zonelist, 0, MAX_NR_ZONES - 1);
  2996. /*
  2997. * Now we build the zonelist so that it contains the zones
  2998. * of all the other nodes.
  2999. * We don't want to pressure a particular node, so when
  3000. * building the zones for node N, we make sure that the
  3001. * zones coming right after the local ones are those from
  3002. * node N+1 (modulo N)
  3003. */
  3004. for (node = local_node + 1; node < MAX_NUMNODES; node++) {
  3005. if (!node_online(node))
  3006. continue;
  3007. j = build_zonelists_node(NODE_DATA(node), zonelist, j,
  3008. MAX_NR_ZONES - 1);
  3009. }
  3010. for (node = 0; node < local_node; node++) {
  3011. if (!node_online(node))
  3012. continue;
  3013. j = build_zonelists_node(NODE_DATA(node), zonelist, j,
  3014. MAX_NR_ZONES - 1);
  3015. }
  3016. zonelist->_zonerefs[j].zone = NULL;
  3017. zonelist->_zonerefs[j].zone_idx = 0;
  3018. }
  3019. /* non-NUMA variant of zonelist performance cache - just NULL zlcache_ptr */
  3020. static void build_zonelist_cache(pg_data_t *pgdat)
  3021. {
  3022. pgdat->node_zonelists[0].zlcache_ptr = NULL;
  3023. }
  3024. #endif /* CONFIG_NUMA */
  3025. /*
  3026. * Boot pageset table. One per cpu which is going to be used for all
  3027. * zones and all nodes. The parameters will be set in such a way
  3028. * that an item put on a list will immediately be handed over to
  3029. * the buddy list. This is safe since pageset manipulation is done
  3030. * with interrupts disabled.
  3031. *
  3032. * The boot_pagesets must be kept even after bootup is complete for
  3033. * unused processors and/or zones. They do play a role for bootstrapping
  3034. * hotplugged processors.
  3035. *
  3036. * zoneinfo_show() and maybe other functions do
  3037. * not check if the processor is online before following the pageset pointer.
  3038. * Other parts of the kernel may not check if the zone is available.
  3039. */
  3040. static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
  3041. static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
  3042. static void setup_zone_pageset(struct zone *zone);
  3043. /*
  3044. * Global mutex to protect against size modification of zonelists
  3045. * as well as to serialize pageset setup for the new populated zone.
  3046. */
  3047. DEFINE_MUTEX(zonelists_mutex);
  3048. /* return values int ....just for stop_machine() */
  3049. static int __build_all_zonelists(void *data)
  3050. {
  3051. int nid;
  3052. int cpu;
  3053. pg_data_t *self = data;
  3054. #ifdef CONFIG_NUMA
  3055. memset(node_load, 0, sizeof(node_load));
  3056. #endif
  3057. if (self && !node_online(self->node_id)) {
  3058. build_zonelists(self);
  3059. build_zonelist_cache(self);
  3060. }
  3061. for_each_online_node(nid) {
  3062. pg_data_t *pgdat = NODE_DATA(nid);
  3063. build_zonelists(pgdat);
  3064. build_zonelist_cache(pgdat);
  3065. }
  3066. /*
  3067. * Initialize the boot_pagesets that are going to be used
  3068. * for bootstrapping processors. The real pagesets for
  3069. * each zone will be allocated later when the per cpu
  3070. * allocator is available.
  3071. *
  3072. * boot_pagesets are used also for bootstrapping offline
  3073. * cpus if the system is already booted because the pagesets
  3074. * are needed to initialize allocators on a specific cpu too.
  3075. * F.e. the percpu allocator needs the page allocator which
  3076. * needs the percpu allocator in order to allocate its pagesets
  3077. * (a chicken-egg dilemma).
  3078. */
  3079. for_each_possible_cpu(cpu) {
  3080. setup_pageset(&per_cpu(boot_pageset, cpu), 0);
  3081. #ifdef CONFIG_HAVE_MEMORYLESS_NODES
  3082. /*
  3083. * We now know the "local memory node" for each node--
  3084. * i.e., the node of the first zone in the generic zonelist.
  3085. * Set up numa_mem percpu variable for on-line cpus. During
  3086. * boot, only the boot cpu should be on-line; we'll init the
  3087. * secondary cpus' numa_mem as they come on-line. During
  3088. * node/memory hotplug, we'll fixup all on-line cpus.
  3089. */
  3090. if (cpu_online(cpu))
  3091. set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu)));
  3092. #endif
  3093. }
  3094. return 0;
  3095. }
  3096. /*
  3097. * Called with zonelists_mutex held always
  3098. * unless system_state == SYSTEM_BOOTING.
  3099. */
  3100. void __ref build_all_zonelists(pg_data_t *pgdat, struct zone *zone)
  3101. {
  3102. set_zonelist_order();
  3103. if (system_state == SYSTEM_BOOTING) {
  3104. __build_all_zonelists(NULL);
  3105. mminit_verify_zonelist();
  3106. cpuset_init_current_mems_allowed();
  3107. } else {
  3108. /* we have to stop all cpus to guarantee there is no user
  3109. of zonelist */
  3110. #ifdef CONFIG_MEMORY_HOTPLUG
  3111. if (zone)
  3112. setup_zone_pageset(zone);
  3113. #endif
  3114. stop_machine(__build_all_zonelists, pgdat, NULL);
  3115. /* cpuset refresh routine should be here */
  3116. }
  3117. vm_total_pages = nr_free_pagecache_pages();
  3118. /*
  3119. * Disable grouping by mobility if the number of pages in the
  3120. * system is too low to allow the mechanism to work. It would be
  3121. * more accurate, but expensive to check per-zone. This check is
  3122. * made on memory-hotadd so a system can start with mobility
  3123. * disabled and enable it later
  3124. */
  3125. if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
  3126. page_group_by_mobility_disabled = 1;
  3127. else
  3128. page_group_by_mobility_disabled = 0;
  3129. printk("Built %i zonelists in %s order, mobility grouping %s. "
  3130. "Total pages: %ld\n",
  3131. nr_online_nodes,
  3132. zonelist_order_name[current_zonelist_order],
  3133. page_group_by_mobility_disabled ? "off" : "on",
  3134. vm_total_pages);
  3135. #ifdef CONFIG_NUMA
  3136. printk("Policy zone: %s\n", zone_names[policy_zone]);
  3137. #endif
  3138. }
  3139. /*
  3140. * Helper functions to size the waitqueue hash table.
  3141. * Essentially these want to choose hash table sizes sufficiently
  3142. * large so that collisions trying to wait on pages are rare.
  3143. * But in fact, the number of active page waitqueues on typical
  3144. * systems is ridiculously low, less than 200. So this is even
  3145. * conservative, even though it seems large.
  3146. *
  3147. * The constant PAGES_PER_WAITQUEUE specifies the ratio of pages to
  3148. * waitqueues, i.e. the size of the waitq table given the number of pages.
  3149. */
  3150. #define PAGES_PER_WAITQUEUE 256
  3151. #ifndef CONFIG_MEMORY_HOTPLUG
  3152. static inline unsigned long wait_table_hash_nr_entries(unsigned long pages)
  3153. {
  3154. unsigned long size = 1;
  3155. pages /= PAGES_PER_WAITQUEUE;
  3156. while (size < pages)
  3157. size <<= 1;
  3158. /*
  3159. * Once we have dozens or even hundreds of threads sleeping
  3160. * on IO we've got bigger problems than wait queue collision.
  3161. * Limit the size of the wait table to a reasonable size.
  3162. */
  3163. size = min(size, 4096UL);
  3164. return max(size, 4UL);
  3165. }
  3166. #else
  3167. /*
  3168. * A zone's size might be changed by hot-add, so it is not possible to determine
  3169. * a suitable size for its wait_table. So we use the maximum size now.
  3170. *
  3171. * The max wait table size = 4096 x sizeof(wait_queue_head_t). ie:
  3172. *
  3173. * i386 (preemption config) : 4096 x 16 = 64Kbyte.
  3174. * ia64, x86-64 (no preemption): 4096 x 20 = 80Kbyte.
  3175. * ia64, x86-64 (preemption) : 4096 x 24 = 96Kbyte.
  3176. *
  3177. * The maximum entries are prepared when a zone's memory is (512K + 256) pages
  3178. * or more by the traditional way. (See above). It equals:
  3179. *
  3180. * i386, x86-64, powerpc(4K page size) : = ( 2G + 1M)byte.
  3181. * ia64(16K page size) : = ( 8G + 4M)byte.
  3182. * powerpc (64K page size) : = (32G +16M)byte.
  3183. */
  3184. static inline unsigned long wait_table_hash_nr_entries(unsigned long pages)
  3185. {
  3186. return 4096UL;
  3187. }
  3188. #endif
  3189. /*
  3190. * This is an integer logarithm so that shifts can be used later
  3191. * to extract the more random high bits from the multiplicative
  3192. * hash function before the remainder is taken.
  3193. */
  3194. static inline unsigned long wait_table_bits(unsigned long size)
  3195. {
  3196. return ffz(~size);
  3197. }
  3198. #define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1))
  3199. /*
  3200. * Check if a pageblock contains reserved pages
  3201. */
  3202. static int pageblock_is_reserved(unsigned long start_pfn, unsigned long end_pfn)
  3203. {
  3204. unsigned long pfn;
  3205. for (pfn = start_pfn; pfn < end_pfn; pfn++) {
  3206. if (!pfn_valid_within(pfn) || PageReserved(pfn_to_page(pfn)))
  3207. return 1;
  3208. }
  3209. return 0;
  3210. }
  3211. /*
  3212. * Mark a number of pageblocks as MIGRATE_RESERVE. The number
  3213. * of blocks reserved is based on min_wmark_pages(zone). The memory within
  3214. * the reserve will tend to store contiguous free pages. Setting min_free_kbytes
  3215. * higher will lead to a bigger reserve which will get freed as contiguous
  3216. * blocks as reclaim kicks in
  3217. */
  3218. static void setup_zone_migrate_reserve(struct zone *zone)
  3219. {
  3220. unsigned long start_pfn, pfn, end_pfn, block_end_pfn;
  3221. struct page *page;
  3222. unsigned long block_migratetype;
  3223. int reserve;
  3224. /*
  3225. * Get the start pfn, end pfn and the number of blocks to reserve
  3226. * We have to be careful to be aligned to pageblock_nr_pages to
  3227. * make sure that we always check pfn_valid for the first page in
  3228. * the block.
  3229. */
  3230. start_pfn = zone->zone_start_pfn;
  3231. end_pfn = start_pfn + zone->spanned_pages;
  3232. start_pfn = roundup(start_pfn, pageblock_nr_pages);
  3233. reserve = roundup(min_wmark_pages(zone), pageblock_nr_pages) >>
  3234. pageblock_order;
  3235. /*
  3236. * Reserve blocks are generally in place to help high-order atomic
  3237. * allocations that are short-lived. A min_free_kbytes value that
  3238. * would result in more than 2 reserve blocks for atomic allocations
  3239. * is assumed to be in place to help anti-fragmentation for the
  3240. * future allocation of hugepages at runtime.
  3241. */
  3242. reserve = min(2, reserve);
  3243. for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
  3244. if (!pfn_valid(pfn))
  3245. continue;
  3246. page = pfn_to_page(pfn);
  3247. /* Watch out for overlapping nodes */
  3248. if (page_to_nid(page) != zone_to_nid(zone))
  3249. continue;
  3250. block_migratetype = get_pageblock_migratetype(page);
  3251. /* Only test what is necessary when the reserves are not met */
  3252. if (reserve > 0) {
  3253. /*
  3254. * Blocks with reserved pages will never free, skip
  3255. * them.
  3256. */
  3257. block_end_pfn = min(pfn + pageblock_nr_pages, end_pfn);
  3258. if (pageblock_is_reserved(pfn, block_end_pfn))
  3259. continue;
  3260. /* If this block is reserved, account for it */
  3261. if (block_migratetype == MIGRATE_RESERVE) {
  3262. reserve--;
  3263. continue;
  3264. }
  3265. /* Suitable for reserving if this block is movable */
  3266. if (block_migratetype == MIGRATE_MOVABLE) {
  3267. set_pageblock_migratetype(page,
  3268. MIGRATE_RESERVE);
  3269. move_freepages_block(zone, page,
  3270. MIGRATE_RESERVE);
  3271. reserve--;
  3272. continue;
  3273. }
  3274. }
  3275. /*
  3276. * If the reserve is met and this is a previous reserved block,
  3277. * take it back
  3278. */
  3279. if (block_migratetype == MIGRATE_RESERVE) {
  3280. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  3281. move_freepages_block(zone, page, MIGRATE_MOVABLE);
  3282. }
  3283. }
  3284. }
  3285. /*
  3286. * Initially all pages are reserved - free ones are freed
  3287. * up by free_all_bootmem() once the early boot process is
  3288. * done. Non-atomic initialization, single-pass.
  3289. */
  3290. void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
  3291. unsigned long start_pfn, enum memmap_context context)
  3292. {
  3293. struct page *page;
  3294. unsigned long end_pfn = start_pfn + size;
  3295. unsigned long pfn;
  3296. struct zone *z;
  3297. if (highest_memmap_pfn < end_pfn - 1)
  3298. highest_memmap_pfn = end_pfn - 1;
  3299. z = &NODE_DATA(nid)->node_zones[zone];
  3300. for (pfn = start_pfn; pfn < end_pfn; pfn++) {
  3301. /*
  3302. * There can be holes in boot-time mem_map[]s
  3303. * handed to this function. They do not
  3304. * exist on hotplugged memory.
  3305. */
  3306. if (context == MEMMAP_EARLY) {
  3307. if (!early_pfn_valid(pfn))
  3308. continue;
  3309. if (!early_pfn_in_nid(pfn, nid))
  3310. continue;
  3311. }
  3312. page = pfn_to_page(pfn);
  3313. set_page_links(page, zone, nid, pfn);
  3314. mminit_verify_page_links(page, zone, nid, pfn);
  3315. init_page_count(page);
  3316. reset_page_mapcount(page);
  3317. SetPageReserved(page);
  3318. /*
  3319. * Mark the block movable so that blocks are reserved for
  3320. * movable at startup. This will force kernel allocations
  3321. * to reserve their blocks rather than leaking throughout
  3322. * the address space during boot when many long-lived
  3323. * kernel allocations are made. Later some blocks near
  3324. * the start are marked MIGRATE_RESERVE by
  3325. * setup_zone_migrate_reserve()
  3326. *
  3327. * bitmap is created for zone's valid pfn range. but memmap
  3328. * can be created for invalid pages (for alignment)
  3329. * check here not to call set_pageblock_migratetype() against
  3330. * pfn out of zone.
  3331. */
  3332. if ((z->zone_start_pfn <= pfn)
  3333. && (pfn < z->zone_start_pfn + z->spanned_pages)
  3334. && !(pfn & (pageblock_nr_pages - 1)))
  3335. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  3336. INIT_LIST_HEAD(&page->lru);
  3337. #ifdef WANT_PAGE_VIRTUAL
  3338. /* The shift won't overflow because ZONE_NORMAL is below 4G. */
  3339. if (!is_highmem_idx(zone))
  3340. set_page_address(page, __va(pfn << PAGE_SHIFT));
  3341. #endif
  3342. }
  3343. }
  3344. static void __meminit zone_init_free_lists(struct zone *zone)
  3345. {
  3346. int order, t;
  3347. for_each_migratetype_order(order, t) {
  3348. INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
  3349. zone->free_area[order].nr_free = 0;
  3350. }
  3351. }
  3352. #ifndef __HAVE_ARCH_MEMMAP_INIT
  3353. #define memmap_init(size, nid, zone, start_pfn) \
  3354. memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY)
  3355. #endif
  3356. static int __meminit zone_batchsize(struct zone *zone)
  3357. {
  3358. #ifdef CONFIG_MMU
  3359. int batch;
  3360. /*
  3361. * The per-cpu-pages pools are set to around 1000th of the
  3362. * size of the zone. But no more than 1/2 of a meg.
  3363. *
  3364. * OK, so we don't know how big the cache is. So guess.
  3365. */
  3366. batch = zone->present_pages / 1024;
  3367. if (batch * PAGE_SIZE > 512 * 1024)
  3368. batch = (512 * 1024) / PAGE_SIZE;
  3369. batch /= 4; /* We effectively *= 4 below */
  3370. if (batch < 1)
  3371. batch = 1;
  3372. /*
  3373. * Clamp the batch to a 2^n - 1 value. Having a power
  3374. * of 2 value was found to be more likely to have
  3375. * suboptimal cache aliasing properties in some cases.
  3376. *
  3377. * For example if 2 tasks are alternately allocating
  3378. * batches of pages, one task can end up with a lot
  3379. * of pages of one half of the possible page colors
  3380. * and the other with pages of the other colors.
  3381. */
  3382. batch = rounddown_pow_of_two(batch + batch/2) - 1;
  3383. return batch;
  3384. #else
  3385. /* The deferral and batching of frees should be suppressed under NOMMU
  3386. * conditions.
  3387. *
  3388. * The problem is that NOMMU needs to be able to allocate large chunks
  3389. * of contiguous memory as there's no hardware page translation to
  3390. * assemble apparent contiguous memory from discontiguous pages.
  3391. *
  3392. * Queueing large contiguous runs of pages for batching, however,
  3393. * causes the pages to actually be freed in smaller chunks. As there
  3394. * can be a significant delay between the individual batches being
  3395. * recycled, this leads to the once large chunks of space being
  3396. * fragmented and becoming unavailable for high-order allocations.
  3397. */
  3398. return 0;
  3399. #endif
  3400. }
  3401. static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
  3402. {
  3403. struct per_cpu_pages *pcp;
  3404. int migratetype;
  3405. memset(p, 0, sizeof(*p));
  3406. pcp = &p->pcp;
  3407. pcp->count = 0;
  3408. pcp->high = 6 * batch;
  3409. pcp->batch = max(1UL, 1 * batch);
  3410. for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++)
  3411. INIT_LIST_HEAD(&pcp->lists[migratetype]);
  3412. }
  3413. /*
  3414. * setup_pagelist_highmark() sets the high water mark for hot per_cpu_pagelist
  3415. * to the value high for the pageset p.
  3416. */
  3417. static void setup_pagelist_highmark(struct per_cpu_pageset *p,
  3418. unsigned long high)
  3419. {
  3420. struct per_cpu_pages *pcp;
  3421. pcp = &p->pcp;
  3422. pcp->high = high;
  3423. pcp->batch = max(1UL, high/4);
  3424. if ((high/4) > (PAGE_SHIFT * 8))
  3425. pcp->batch = PAGE_SHIFT * 8;
  3426. }
  3427. static void __meminit setup_zone_pageset(struct zone *zone)
  3428. {
  3429. int cpu;
  3430. zone->pageset = alloc_percpu(struct per_cpu_pageset);
  3431. for_each_possible_cpu(cpu) {
  3432. struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);
  3433. setup_pageset(pcp, zone_batchsize(zone));
  3434. if (percpu_pagelist_fraction)
  3435. setup_pagelist_highmark(pcp,
  3436. (zone->present_pages /
  3437. percpu_pagelist_fraction));
  3438. }
  3439. }
  3440. /*
  3441. * Allocate per cpu pagesets and initialize them.
  3442. * Before this call only boot pagesets were available.
  3443. */
  3444. void __init setup_per_cpu_pageset(void)
  3445. {
  3446. struct zone *zone;
  3447. for_each_populated_zone(zone)
  3448. setup_zone_pageset(zone);
  3449. }
  3450. static noinline __init_refok
  3451. int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages)
  3452. {
  3453. int i;
  3454. struct pglist_data *pgdat = zone->zone_pgdat;
  3455. size_t alloc_size;
  3456. /*
  3457. * The per-page waitqueue mechanism uses hashed waitqueues
  3458. * per zone.
  3459. */
  3460. zone->wait_table_hash_nr_entries =
  3461. wait_table_hash_nr_entries(zone_size_pages);
  3462. zone->wait_table_bits =
  3463. wait_table_bits(zone->wait_table_hash_nr_entries);
  3464. alloc_size = zone->wait_table_hash_nr_entries
  3465. * sizeof(wait_queue_head_t);
  3466. if (!slab_is_available()) {
  3467. zone->wait_table = (wait_queue_head_t *)
  3468. alloc_bootmem_node_nopanic(pgdat, alloc_size);
  3469. } else {
  3470. /*
  3471. * This case means that a zone whose size was 0 gets new memory
  3472. * via memory hot-add.
  3473. * But it may be the case that a new node was hot-added. In
  3474. * this case vmalloc() will not be able to use this new node's
  3475. * memory - this wait_table must be initialized to use this new
  3476. * node itself as well.
  3477. * To use this new node's memory, further consideration will be
  3478. * necessary.
  3479. */
  3480. zone->wait_table = vmalloc(alloc_size);
  3481. }
  3482. if (!zone->wait_table)
  3483. return -ENOMEM;
  3484. for(i = 0; i < zone->wait_table_hash_nr_entries; ++i)
  3485. init_waitqueue_head(zone->wait_table + i);
  3486. return 0;
  3487. }
  3488. static __meminit void zone_pcp_init(struct zone *zone)
  3489. {
  3490. /*
  3491. * per cpu subsystem is not up at this point. The following code
  3492. * relies on the ability of the linker to provide the
  3493. * offset of a (static) per cpu variable into the per cpu area.
  3494. */
  3495. zone->pageset = &boot_pageset;
  3496. if (zone->present_pages)
  3497. printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n",
  3498. zone->name, zone->present_pages,
  3499. zone_batchsize(zone));
  3500. }
  3501. int __meminit init_currently_empty_zone(struct zone *zone,
  3502. unsigned long zone_start_pfn,
  3503. unsigned long size,
  3504. enum memmap_context context)
  3505. {
  3506. struct pglist_data *pgdat = zone->zone_pgdat;
  3507. int ret;
  3508. ret = zone_wait_table_init(zone, size);
  3509. if (ret)
  3510. return ret;
  3511. pgdat->nr_zones = zone_idx(zone) + 1;
  3512. zone->zone_start_pfn = zone_start_pfn;
  3513. mminit_dprintk(MMINIT_TRACE, "memmap_init",
  3514. "Initialising map node %d zone %lu pfns %lu -> %lu\n",
  3515. pgdat->node_id,
  3516. (unsigned long)zone_idx(zone),
  3517. zone_start_pfn, (zone_start_pfn + size));
  3518. zone_init_free_lists(zone);
  3519. return 0;
  3520. }
  3521. #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
  3522. #ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
  3523. /*
  3524. * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
  3525. * Architectures may implement their own version but if add_active_range()
  3526. * was used and there are no special requirements, this is a convenient
  3527. * alternative
  3528. */
  3529. int __meminit __early_pfn_to_nid(unsigned long pfn)
  3530. {
  3531. unsigned long start_pfn, end_pfn;
  3532. int i, nid;
  3533. for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
  3534. if (start_pfn <= pfn && pfn < end_pfn)
  3535. return nid;
  3536. /* This is a memory hole */
  3537. return -1;
  3538. }
  3539. #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
  3540. int __meminit early_pfn_to_nid(unsigned long pfn)
  3541. {
  3542. int nid;
  3543. nid = __early_pfn_to_nid(pfn);
  3544. if (nid >= 0)
  3545. return nid;
  3546. /* just returns 0 */
  3547. return 0;
  3548. }
  3549. #ifdef CONFIG_NODES_SPAN_OTHER_NODES
  3550. bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
  3551. {
  3552. int nid;
  3553. nid = __early_pfn_to_nid(pfn);
  3554. if (nid >= 0 && nid != node)
  3555. return false;
  3556. return true;
  3557. }
  3558. #endif
  3559. /**
  3560. * free_bootmem_with_active_regions - Call free_bootmem_node for each active range
  3561. * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed.
  3562. * @max_low_pfn: The highest PFN that will be passed to free_bootmem_node
  3563. *
  3564. * If an architecture guarantees that all ranges registered with
  3565. * add_active_ranges() contain no holes and may be freed, this
  3566. * this function may be used instead of calling free_bootmem() manually.
  3567. */
  3568. void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn)
  3569. {
  3570. unsigned long start_pfn, end_pfn;
  3571. int i, this_nid;
  3572. for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) {
  3573. start_pfn = min(start_pfn, max_low_pfn);
  3574. end_pfn = min(end_pfn, max_low_pfn);
  3575. if (start_pfn < end_pfn)
  3576. free_bootmem_node(NODE_DATA(this_nid),
  3577. PFN_PHYS(start_pfn),
  3578. (end_pfn - start_pfn) << PAGE_SHIFT);
  3579. }
  3580. }
  3581. /**
  3582. * sparse_memory_present_with_active_regions - Call memory_present for each active range
  3583. * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used.
  3584. *
  3585. * If an architecture guarantees that all ranges registered with
  3586. * add_active_ranges() contain no holes and may be freed, this
  3587. * function may be used instead of calling memory_present() manually.
  3588. */
  3589. void __init sparse_memory_present_with_active_regions(int nid)
  3590. {
  3591. unsigned long start_pfn, end_pfn;
  3592. int i, this_nid;
  3593. for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid)
  3594. memory_present(this_nid, start_pfn, end_pfn);
  3595. }
  3596. /**
  3597. * get_pfn_range_for_nid - Return the start and end page frames for a node
  3598. * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
  3599. * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
  3600. * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
  3601. *
  3602. * It returns the start and end page frame of a node based on information
  3603. * provided by an arch calling add_active_range(). If called for a node
  3604. * with no available memory, a warning is printed and the start and end
  3605. * PFNs will be 0.
  3606. */
  3607. void __meminit get_pfn_range_for_nid(unsigned int nid,
  3608. unsigned long *start_pfn, unsigned long *end_pfn)
  3609. {
  3610. unsigned long this_start_pfn, this_end_pfn;
  3611. int i;
  3612. *start_pfn = -1UL;
  3613. *end_pfn = 0;
  3614. for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) {
  3615. *start_pfn = min(*start_pfn, this_start_pfn);
  3616. *end_pfn = max(*end_pfn, this_end_pfn);
  3617. }
  3618. if (*start_pfn == -1UL)
  3619. *start_pfn = 0;
  3620. }
  3621. /*
  3622. * This finds a zone that can be used for ZONE_MOVABLE pages. The
  3623. * assumption is made that zones within a node are ordered in monotonic
  3624. * increasing memory addresses so that the "highest" populated zone is used
  3625. */
  3626. static void __init find_usable_zone_for_movable(void)
  3627. {
  3628. int zone_index;
  3629. for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
  3630. if (zone_index == ZONE_MOVABLE)
  3631. continue;
  3632. if (arch_zone_highest_possible_pfn[zone_index] >
  3633. arch_zone_lowest_possible_pfn[zone_index])
  3634. break;
  3635. }
  3636. VM_BUG_ON(zone_index == -1);
  3637. movable_zone = zone_index;
  3638. }
  3639. /*
  3640. * The zone ranges provided by the architecture do not include ZONE_MOVABLE
  3641. * because it is sized independent of architecture. Unlike the other zones,
  3642. * the starting point for ZONE_MOVABLE is not fixed. It may be different
  3643. * in each node depending on the size of each node and how evenly kernelcore
  3644. * is distributed. This helper function adjusts the zone ranges
  3645. * provided by the architecture for a given node by using the end of the
  3646. * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
  3647. * zones within a node are in order of monotonic increases memory addresses
  3648. */
  3649. static void __meminit adjust_zone_range_for_zone_movable(int nid,
  3650. unsigned long zone_type,
  3651. unsigned long node_start_pfn,
  3652. unsigned long node_end_pfn,
  3653. unsigned long *zone_start_pfn,
  3654. unsigned long *zone_end_pfn)
  3655. {
  3656. /* Only adjust if ZONE_MOVABLE is on this node */
  3657. if (zone_movable_pfn[nid]) {
  3658. /* Size ZONE_MOVABLE */
  3659. if (zone_type == ZONE_MOVABLE) {
  3660. *zone_start_pfn = zone_movable_pfn[nid];
  3661. *zone_end_pfn = min(node_end_pfn,
  3662. arch_zone_highest_possible_pfn[movable_zone]);
  3663. /* Adjust for ZONE_MOVABLE starting within this range */
  3664. } else if (*zone_start_pfn < zone_movable_pfn[nid] &&
  3665. *zone_end_pfn > zone_movable_pfn[nid]) {
  3666. *zone_end_pfn = zone_movable_pfn[nid];
  3667. /* Check if this whole range is within ZONE_MOVABLE */
  3668. } else if (*zone_start_pfn >= zone_movable_pfn[nid])
  3669. *zone_start_pfn = *zone_end_pfn;
  3670. }
  3671. }
  3672. /*
  3673. * Return the number of pages a zone spans in a node, including holes
  3674. * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
  3675. */
  3676. static unsigned long __meminit zone_spanned_pages_in_node(int nid,
  3677. unsigned long zone_type,
  3678. unsigned long *ignored)
  3679. {
  3680. unsigned long node_start_pfn, node_end_pfn;
  3681. unsigned long zone_start_pfn, zone_end_pfn;
  3682. /* Get the start and end of the node and zone */
  3683. get_pfn_range_for_nid(nid, &node_start_pfn, &node_end_pfn);
  3684. zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type];
  3685. zone_end_pfn = arch_zone_highest_possible_pfn[zone_type];
  3686. adjust_zone_range_for_zone_movable(nid, zone_type,
  3687. node_start_pfn, node_end_pfn,
  3688. &zone_start_pfn, &zone_end_pfn);
  3689. /* Check that this node has pages within the zone's required range */
  3690. if (zone_end_pfn < node_start_pfn || zone_start_pfn > node_end_pfn)
  3691. return 0;
  3692. /* Move the zone boundaries inside the node if necessary */
  3693. zone_end_pfn = min(zone_end_pfn, node_end_pfn);
  3694. zone_start_pfn = max(zone_start_pfn, node_start_pfn);
  3695. /* Return the spanned pages */
  3696. return zone_end_pfn - zone_start_pfn;
  3697. }
  3698. /*
  3699. * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
  3700. * then all holes in the requested range will be accounted for.
  3701. */
  3702. unsigned long __meminit __absent_pages_in_range(int nid,
  3703. unsigned long range_start_pfn,
  3704. unsigned long range_end_pfn)
  3705. {
  3706. unsigned long nr_absent = range_end_pfn - range_start_pfn;
  3707. unsigned long start_pfn, end_pfn;
  3708. int i;
  3709. for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
  3710. start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
  3711. end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
  3712. nr_absent -= end_pfn - start_pfn;
  3713. }
  3714. return nr_absent;
  3715. }
  3716. /**
  3717. * absent_pages_in_range - Return number of page frames in holes within a range
  3718. * @start_pfn: The start PFN to start searching for holes
  3719. * @end_pfn: The end PFN to stop searching for holes
  3720. *
  3721. * It returns the number of pages frames in memory holes within a range.
  3722. */
  3723. unsigned long __init absent_pages_in_range(unsigned long start_pfn,
  3724. unsigned long end_pfn)
  3725. {
  3726. return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
  3727. }
  3728. /* Return the number of page frames in holes in a zone on a node */
  3729. static unsigned long __meminit zone_absent_pages_in_node(int nid,
  3730. unsigned long zone_type,
  3731. unsigned long *ignored)
  3732. {
  3733. unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
  3734. unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
  3735. unsigned long node_start_pfn, node_end_pfn;
  3736. unsigned long zone_start_pfn, zone_end_pfn;
  3737. get_pfn_range_for_nid(nid, &node_start_pfn, &node_end_pfn);
  3738. zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
  3739. zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
  3740. adjust_zone_range_for_zone_movable(nid, zone_type,
  3741. node_start_pfn, node_end_pfn,
  3742. &zone_start_pfn, &zone_end_pfn);
  3743. return __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
  3744. }
  3745. #else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
  3746. static inline unsigned long __meminit zone_spanned_pages_in_node(int nid,
  3747. unsigned long zone_type,
  3748. unsigned long *zones_size)
  3749. {
  3750. return zones_size[zone_type];
  3751. }
  3752. static inline unsigned long __meminit zone_absent_pages_in_node(int nid,
  3753. unsigned long zone_type,
  3754. unsigned long *zholes_size)
  3755. {
  3756. if (!zholes_size)
  3757. return 0;
  3758. return zholes_size[zone_type];
  3759. }
  3760. #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
  3761. static void __meminit calculate_node_totalpages(struct pglist_data *pgdat,
  3762. unsigned long *zones_size, unsigned long *zholes_size)
  3763. {
  3764. unsigned long realtotalpages, totalpages = 0;
  3765. enum zone_type i;
  3766. for (i = 0; i < MAX_NR_ZONES; i++)
  3767. totalpages += zone_spanned_pages_in_node(pgdat->node_id, i,
  3768. zones_size);
  3769. pgdat->node_spanned_pages = totalpages;
  3770. realtotalpages = totalpages;
  3771. for (i = 0; i < MAX_NR_ZONES; i++)
  3772. realtotalpages -=
  3773. zone_absent_pages_in_node(pgdat->node_id, i,
  3774. zholes_size);
  3775. pgdat->node_present_pages = realtotalpages;
  3776. printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id,
  3777. realtotalpages);
  3778. }
  3779. #ifndef CONFIG_SPARSEMEM
  3780. /*
  3781. * Calculate the size of the zone->blockflags rounded to an unsigned long
  3782. * Start by making sure zonesize is a multiple of pageblock_order by rounding
  3783. * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
  3784. * round what is now in bits to nearest long in bits, then return it in
  3785. * bytes.
  3786. */
  3787. static unsigned long __init usemap_size(unsigned long zonesize)
  3788. {
  3789. unsigned long usemapsize;
  3790. usemapsize = roundup(zonesize, pageblock_nr_pages);
  3791. usemapsize = usemapsize >> pageblock_order;
  3792. usemapsize *= NR_PAGEBLOCK_BITS;
  3793. usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
  3794. return usemapsize / 8;
  3795. }
  3796. static void __init setup_usemap(struct pglist_data *pgdat,
  3797. struct zone *zone, unsigned long zonesize)
  3798. {
  3799. unsigned long usemapsize = usemap_size(zonesize);
  3800. zone->pageblock_flags = NULL;
  3801. if (usemapsize)
  3802. zone->pageblock_flags = alloc_bootmem_node_nopanic(pgdat,
  3803. usemapsize);
  3804. }
  3805. #else
  3806. static inline void setup_usemap(struct pglist_data *pgdat,
  3807. struct zone *zone, unsigned long zonesize) {}
  3808. #endif /* CONFIG_SPARSEMEM */
  3809. #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
  3810. /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
  3811. void __init set_pageblock_order(void)
  3812. {
  3813. unsigned int order;
  3814. /* Check that pageblock_nr_pages has not already been setup */
  3815. if (pageblock_order)
  3816. return;
  3817. if (HPAGE_SHIFT > PAGE_SHIFT)
  3818. order = HUGETLB_PAGE_ORDER;
  3819. else
  3820. order = MAX_ORDER - 1;
  3821. /*
  3822. * Assume the largest contiguous order of interest is a huge page.
  3823. * This value may be variable depending on boot parameters on IA64 and
  3824. * powerpc.
  3825. */
  3826. pageblock_order = order;
  3827. }
  3828. #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
  3829. /*
  3830. * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
  3831. * is unused as pageblock_order is set at compile-time. See
  3832. * include/linux/pageblock-flags.h for the values of pageblock_order based on
  3833. * the kernel config
  3834. */
  3835. void __init set_pageblock_order(void)
  3836. {
  3837. }
  3838. #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
  3839. /*
  3840. * Set up the zone data structures:
  3841. * - mark all pages reserved
  3842. * - mark all memory queues empty
  3843. * - clear the memory bitmaps
  3844. *
  3845. * NOTE: pgdat should get zeroed by caller.
  3846. */
  3847. static void __paginginit free_area_init_core(struct pglist_data *pgdat,
  3848. unsigned long *zones_size, unsigned long *zholes_size)
  3849. {
  3850. enum zone_type j;
  3851. int nid = pgdat->node_id;
  3852. unsigned long zone_start_pfn = pgdat->node_start_pfn;
  3853. int ret;
  3854. pgdat_resize_init(pgdat);
  3855. init_waitqueue_head(&pgdat->kswapd_wait);
  3856. init_waitqueue_head(&pgdat->pfmemalloc_wait);
  3857. pgdat_page_cgroup_init(pgdat);
  3858. for (j = 0; j < MAX_NR_ZONES; j++) {
  3859. struct zone *zone = pgdat->node_zones + j;
  3860. unsigned long size, realsize, memmap_pages;
  3861. size = zone_spanned_pages_in_node(nid, j, zones_size);
  3862. realsize = size - zone_absent_pages_in_node(nid, j,
  3863. zholes_size);
  3864. /*
  3865. * Adjust realsize so that it accounts for how much memory
  3866. * is used by this zone for memmap. This affects the watermark
  3867. * and per-cpu initialisations
  3868. */
  3869. memmap_pages =
  3870. PAGE_ALIGN(size * sizeof(struct page)) >> PAGE_SHIFT;
  3871. if (realsize >= memmap_pages) {
  3872. realsize -= memmap_pages;
  3873. if (memmap_pages)
  3874. printk(KERN_DEBUG
  3875. " %s zone: %lu pages used for memmap\n",
  3876. zone_names[j], memmap_pages);
  3877. } else
  3878. printk(KERN_WARNING
  3879. " %s zone: %lu pages exceeds realsize %lu\n",
  3880. zone_names[j], memmap_pages, realsize);
  3881. /* Account for reserved pages */
  3882. if (j == 0 && realsize > dma_reserve) {
  3883. realsize -= dma_reserve;
  3884. printk(KERN_DEBUG " %s zone: %lu pages reserved\n",
  3885. zone_names[0], dma_reserve);
  3886. }
  3887. if (!is_highmem_idx(j))
  3888. nr_kernel_pages += realsize;
  3889. nr_all_pages += realsize;
  3890. zone->spanned_pages = size;
  3891. zone->present_pages = realsize;
  3892. #ifdef CONFIG_NUMA
  3893. zone->node = nid;
  3894. zone->min_unmapped_pages = (realsize*sysctl_min_unmapped_ratio)
  3895. / 100;
  3896. zone->min_slab_pages = (realsize * sysctl_min_slab_ratio) / 100;
  3897. #endif
  3898. zone->name = zone_names[j];
  3899. spin_lock_init(&zone->lock);
  3900. spin_lock_init(&zone->lru_lock);
  3901. zone_seqlock_init(zone);
  3902. zone->zone_pgdat = pgdat;
  3903. zone_pcp_init(zone);
  3904. lruvec_init(&zone->lruvec, zone);
  3905. if (!size)
  3906. continue;
  3907. set_pageblock_order();
  3908. setup_usemap(pgdat, zone, size);
  3909. ret = init_currently_empty_zone(zone, zone_start_pfn,
  3910. size, MEMMAP_EARLY);
  3911. BUG_ON(ret);
  3912. memmap_init(size, nid, j, zone_start_pfn);
  3913. zone_start_pfn += size;
  3914. }
  3915. }
  3916. static void __init_refok alloc_node_mem_map(struct pglist_data *pgdat)
  3917. {
  3918. /* Skip empty nodes */
  3919. if (!pgdat->node_spanned_pages)
  3920. return;
  3921. #ifdef CONFIG_FLAT_NODE_MEM_MAP
  3922. /* ia64 gets its own node_mem_map, before this, without bootmem */
  3923. if (!pgdat->node_mem_map) {
  3924. unsigned long size, start, end;
  3925. struct page *map;
  3926. /*
  3927. * The zone's endpoints aren't required to be MAX_ORDER
  3928. * aligned but the node_mem_map endpoints must be in order
  3929. * for the buddy allocator to function correctly.
  3930. */
  3931. start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
  3932. end = pgdat->node_start_pfn + pgdat->node_spanned_pages;
  3933. end = ALIGN(end, MAX_ORDER_NR_PAGES);
  3934. size = (end - start) * sizeof(struct page);
  3935. map = alloc_remap(pgdat->node_id, size);
  3936. if (!map)
  3937. map = alloc_bootmem_node_nopanic(pgdat, size);
  3938. pgdat->node_mem_map = map + (pgdat->node_start_pfn - start);
  3939. }
  3940. #ifndef CONFIG_NEED_MULTIPLE_NODES
  3941. /*
  3942. * With no DISCONTIG, the global mem_map is just set as node 0's
  3943. */
  3944. if (pgdat == NODE_DATA(0)) {
  3945. mem_map = NODE_DATA(0)->node_mem_map;
  3946. #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
  3947. if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
  3948. mem_map -= (pgdat->node_start_pfn - ARCH_PFN_OFFSET);
  3949. #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
  3950. }
  3951. #endif
  3952. #endif /* CONFIG_FLAT_NODE_MEM_MAP */
  3953. }
  3954. void __paginginit free_area_init_node(int nid, unsigned long *zones_size,
  3955. unsigned long node_start_pfn, unsigned long *zholes_size)
  3956. {
  3957. pg_data_t *pgdat = NODE_DATA(nid);
  3958. /* pg_data_t should be reset to zero when it's allocated */
  3959. WARN_ON(pgdat->nr_zones || pgdat->classzone_idx);
  3960. pgdat->node_id = nid;
  3961. pgdat->node_start_pfn = node_start_pfn;
  3962. calculate_node_totalpages(pgdat, zones_size, zholes_size);
  3963. alloc_node_mem_map(pgdat);
  3964. #ifdef CONFIG_FLAT_NODE_MEM_MAP
  3965. printk(KERN_DEBUG "free_area_init_node: node %d, pgdat %08lx, node_mem_map %08lx\n",
  3966. nid, (unsigned long)pgdat,
  3967. (unsigned long)pgdat->node_mem_map);
  3968. #endif
  3969. free_area_init_core(pgdat, zones_size, zholes_size);
  3970. }
  3971. #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
  3972. #if MAX_NUMNODES > 1
  3973. /*
  3974. * Figure out the number of possible node ids.
  3975. */
  3976. static void __init setup_nr_node_ids(void)
  3977. {
  3978. unsigned int node;
  3979. unsigned int highest = 0;
  3980. for_each_node_mask(node, node_possible_map)
  3981. highest = node;
  3982. nr_node_ids = highest + 1;
  3983. }
  3984. #else
  3985. static inline void setup_nr_node_ids(void)
  3986. {
  3987. }
  3988. #endif
  3989. /**
  3990. * node_map_pfn_alignment - determine the maximum internode alignment
  3991. *
  3992. * This function should be called after node map is populated and sorted.
  3993. * It calculates the maximum power of two alignment which can distinguish
  3994. * all the nodes.
  3995. *
  3996. * For example, if all nodes are 1GiB and aligned to 1GiB, the return value
  3997. * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the
  3998. * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is
  3999. * shifted, 1GiB is enough and this function will indicate so.
  4000. *
  4001. * This is used to test whether pfn -> nid mapping of the chosen memory
  4002. * model has fine enough granularity to avoid incorrect mapping for the
  4003. * populated node map.
  4004. *
  4005. * Returns the determined alignment in pfn's. 0 if there is no alignment
  4006. * requirement (single node).
  4007. */
  4008. unsigned long __init node_map_pfn_alignment(void)
  4009. {
  4010. unsigned long accl_mask = 0, last_end = 0;
  4011. unsigned long start, end, mask;
  4012. int last_nid = -1;
  4013. int i, nid;
  4014. for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {
  4015. if (!start || last_nid < 0 || last_nid == nid) {
  4016. last_nid = nid;
  4017. last_end = end;
  4018. continue;
  4019. }
  4020. /*
  4021. * Start with a mask granular enough to pin-point to the
  4022. * start pfn and tick off bits one-by-one until it becomes
  4023. * too coarse to separate the current node from the last.
  4024. */
  4025. mask = ~((1 << __ffs(start)) - 1);
  4026. while (mask && last_end <= (start & (mask << 1)))
  4027. mask <<= 1;
  4028. /* accumulate all internode masks */
  4029. accl_mask |= mask;
  4030. }
  4031. /* convert mask to number of pages */
  4032. return ~accl_mask + 1;
  4033. }
  4034. /* Find the lowest pfn for a node */
  4035. static unsigned long __init find_min_pfn_for_node(int nid)
  4036. {
  4037. unsigned long min_pfn = ULONG_MAX;
  4038. unsigned long start_pfn;
  4039. int i;
  4040. for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL)
  4041. min_pfn = min(min_pfn, start_pfn);
  4042. if (min_pfn == ULONG_MAX) {
  4043. printk(KERN_WARNING
  4044. "Could not find start_pfn for node %d\n", nid);
  4045. return 0;
  4046. }
  4047. return min_pfn;
  4048. }
  4049. /**
  4050. * find_min_pfn_with_active_regions - Find the minimum PFN registered
  4051. *
  4052. * It returns the minimum PFN based on information provided via
  4053. * add_active_range().
  4054. */
  4055. unsigned long __init find_min_pfn_with_active_regions(void)
  4056. {
  4057. return find_min_pfn_for_node(MAX_NUMNODES);
  4058. }
  4059. /*
  4060. * early_calculate_totalpages()
  4061. * Sum pages in active regions for movable zone.
  4062. * Populate N_HIGH_MEMORY for calculating usable_nodes.
  4063. */
  4064. static unsigned long __init early_calculate_totalpages(void)
  4065. {
  4066. unsigned long totalpages = 0;
  4067. unsigned long start_pfn, end_pfn;
  4068. int i, nid;
  4069. for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
  4070. unsigned long pages = end_pfn - start_pfn;
  4071. totalpages += pages;
  4072. if (pages)
  4073. node_set_state(nid, N_HIGH_MEMORY);
  4074. }
  4075. return totalpages;
  4076. }
  4077. /*
  4078. * Find the PFN the Movable zone begins in each node. Kernel memory
  4079. * is spread evenly between nodes as long as the nodes have enough
  4080. * memory. When they don't, some nodes will have more kernelcore than
  4081. * others
  4082. */
  4083. static void __init find_zone_movable_pfns_for_nodes(void)
  4084. {
  4085. int i, nid;
  4086. unsigned long usable_startpfn;
  4087. unsigned long kernelcore_node, kernelcore_remaining;
  4088. /* save the state before borrow the nodemask */
  4089. nodemask_t saved_node_state = node_states[N_HIGH_MEMORY];
  4090. unsigned long totalpages = early_calculate_totalpages();
  4091. int usable_nodes = nodes_weight(node_states[N_HIGH_MEMORY]);
  4092. /*
  4093. * If movablecore was specified, calculate what size of
  4094. * kernelcore that corresponds so that memory usable for
  4095. * any allocation type is evenly spread. If both kernelcore
  4096. * and movablecore are specified, then the value of kernelcore
  4097. * will be used for required_kernelcore if it's greater than
  4098. * what movablecore would have allowed.
  4099. */
  4100. if (required_movablecore) {
  4101. unsigned long corepages;
  4102. /*
  4103. * Round-up so that ZONE_MOVABLE is at least as large as what
  4104. * was requested by the user
  4105. */
  4106. required_movablecore =
  4107. roundup(required_movablecore, MAX_ORDER_NR_PAGES);
  4108. corepages = totalpages - required_movablecore;
  4109. required_kernelcore = max(required_kernelcore, corepages);
  4110. }
  4111. /* If kernelcore was not specified, there is no ZONE_MOVABLE */
  4112. if (!required_kernelcore)
  4113. goto out;
  4114. /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
  4115. find_usable_zone_for_movable();
  4116. usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
  4117. restart:
  4118. /* Spread kernelcore memory as evenly as possible throughout nodes */
  4119. kernelcore_node = required_kernelcore / usable_nodes;
  4120. for_each_node_state(nid, N_HIGH_MEMORY) {
  4121. unsigned long start_pfn, end_pfn;
  4122. /*
  4123. * Recalculate kernelcore_node if the division per node
  4124. * now exceeds what is necessary to satisfy the requested
  4125. * amount of memory for the kernel
  4126. */
  4127. if (required_kernelcore < kernelcore_node)
  4128. kernelcore_node = required_kernelcore / usable_nodes;
  4129. /*
  4130. * As the map is walked, we track how much memory is usable
  4131. * by the kernel using kernelcore_remaining. When it is
  4132. * 0, the rest of the node is usable by ZONE_MOVABLE
  4133. */
  4134. kernelcore_remaining = kernelcore_node;
  4135. /* Go through each range of PFNs within this node */
  4136. for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
  4137. unsigned long size_pages;
  4138. start_pfn = max(start_pfn, zone_movable_pfn[nid]);
  4139. if (start_pfn >= end_pfn)
  4140. continue;
  4141. /* Account for what is only usable for kernelcore */
  4142. if (start_pfn < usable_startpfn) {
  4143. unsigned long kernel_pages;
  4144. kernel_pages = min(end_pfn, usable_startpfn)
  4145. - start_pfn;
  4146. kernelcore_remaining -= min(kernel_pages,
  4147. kernelcore_remaining);
  4148. required_kernelcore -= min(kernel_pages,
  4149. required_kernelcore);
  4150. /* Continue if range is now fully accounted */
  4151. if (end_pfn <= usable_startpfn) {
  4152. /*
  4153. * Push zone_movable_pfn to the end so
  4154. * that if we have to rebalance
  4155. * kernelcore across nodes, we will
  4156. * not double account here
  4157. */
  4158. zone_movable_pfn[nid] = end_pfn;
  4159. continue;
  4160. }
  4161. start_pfn = usable_startpfn;
  4162. }
  4163. /*
  4164. * The usable PFN range for ZONE_MOVABLE is from
  4165. * start_pfn->end_pfn. Calculate size_pages as the
  4166. * number of pages used as kernelcore
  4167. */
  4168. size_pages = end_pfn - start_pfn;
  4169. if (size_pages > kernelcore_remaining)
  4170. size_pages = kernelcore_remaining;
  4171. zone_movable_pfn[nid] = start_pfn + size_pages;
  4172. /*
  4173. * Some kernelcore has been met, update counts and
  4174. * break if the kernelcore for this node has been
  4175. * satisified
  4176. */
  4177. required_kernelcore -= min(required_kernelcore,
  4178. size_pages);
  4179. kernelcore_remaining -= size_pages;
  4180. if (!kernelcore_remaining)
  4181. break;
  4182. }
  4183. }
  4184. /*
  4185. * If there is still required_kernelcore, we do another pass with one
  4186. * less node in the count. This will push zone_movable_pfn[nid] further
  4187. * along on the nodes that still have memory until kernelcore is
  4188. * satisified
  4189. */
  4190. usable_nodes--;
  4191. if (usable_nodes && required_kernelcore > usable_nodes)
  4192. goto restart;
  4193. /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
  4194. for (nid = 0; nid < MAX_NUMNODES; nid++)
  4195. zone_movable_pfn[nid] =
  4196. roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
  4197. out:
  4198. /* restore the node_state */
  4199. node_states[N_HIGH_MEMORY] = saved_node_state;
  4200. }
  4201. /* Any regular memory on that node ? */
  4202. static void __init check_for_regular_memory(pg_data_t *pgdat)
  4203. {
  4204. #ifdef CONFIG_HIGHMEM
  4205. enum zone_type zone_type;
  4206. for (zone_type = 0; zone_type <= ZONE_NORMAL; zone_type++) {
  4207. struct zone *zone = &pgdat->node_zones[zone_type];
  4208. if (zone->present_pages) {
  4209. node_set_state(zone_to_nid(zone), N_NORMAL_MEMORY);
  4210. break;
  4211. }
  4212. }
  4213. #endif
  4214. }
  4215. /**
  4216. * free_area_init_nodes - Initialise all pg_data_t and zone data
  4217. * @max_zone_pfn: an array of max PFNs for each zone
  4218. *
  4219. * This will call free_area_init_node() for each active node in the system.
  4220. * Using the page ranges provided by add_active_range(), the size of each
  4221. * zone in each node and their holes is calculated. If the maximum PFN
  4222. * between two adjacent zones match, it is assumed that the zone is empty.
  4223. * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
  4224. * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
  4225. * starts where the previous one ended. For example, ZONE_DMA32 starts
  4226. * at arch_max_dma_pfn.
  4227. */
  4228. void __init free_area_init_nodes(unsigned long *max_zone_pfn)
  4229. {
  4230. unsigned long start_pfn, end_pfn;
  4231. int i, nid;
  4232. /* Record where the zone boundaries are */
  4233. memset(arch_zone_lowest_possible_pfn, 0,
  4234. sizeof(arch_zone_lowest_possible_pfn));
  4235. memset(arch_zone_highest_possible_pfn, 0,
  4236. sizeof(arch_zone_highest_possible_pfn));
  4237. arch_zone_lowest_possible_pfn[0] = find_min_pfn_with_active_regions();
  4238. arch_zone_highest_possible_pfn[0] = max_zone_pfn[0];
  4239. for (i = 1; i < MAX_NR_ZONES; i++) {
  4240. if (i == ZONE_MOVABLE)
  4241. continue;
  4242. arch_zone_lowest_possible_pfn[i] =
  4243. arch_zone_highest_possible_pfn[i-1];
  4244. arch_zone_highest_possible_pfn[i] =
  4245. max(max_zone_pfn[i], arch_zone_lowest_possible_pfn[i]);
  4246. }
  4247. arch_zone_lowest_possible_pfn[ZONE_MOVABLE] = 0;
  4248. arch_zone_highest_possible_pfn[ZONE_MOVABLE] = 0;
  4249. /* Find the PFNs that ZONE_MOVABLE begins at in each node */
  4250. memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
  4251. find_zone_movable_pfns_for_nodes();
  4252. /* Print out the zone ranges */
  4253. printk("Zone ranges:\n");
  4254. for (i = 0; i < MAX_NR_ZONES; i++) {
  4255. if (i == ZONE_MOVABLE)
  4256. continue;
  4257. printk(KERN_CONT " %-8s ", zone_names[i]);
  4258. if (arch_zone_lowest_possible_pfn[i] ==
  4259. arch_zone_highest_possible_pfn[i])
  4260. printk(KERN_CONT "empty\n");
  4261. else
  4262. printk(KERN_CONT "[mem %0#10lx-%0#10lx]\n",
  4263. arch_zone_lowest_possible_pfn[i] << PAGE_SHIFT,
  4264. (arch_zone_highest_possible_pfn[i]
  4265. << PAGE_SHIFT) - 1);
  4266. }
  4267. /* Print out the PFNs ZONE_MOVABLE begins at in each node */
  4268. printk("Movable zone start for each node\n");
  4269. for (i = 0; i < MAX_NUMNODES; i++) {
  4270. if (zone_movable_pfn[i])
  4271. printk(" Node %d: %#010lx\n", i,
  4272. zone_movable_pfn[i] << PAGE_SHIFT);
  4273. }
  4274. /* Print out the early node map */
  4275. printk("Early memory node ranges\n");
  4276. for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
  4277. printk(" node %3d: [mem %#010lx-%#010lx]\n", nid,
  4278. start_pfn << PAGE_SHIFT, (end_pfn << PAGE_SHIFT) - 1);
  4279. /* Initialise every node */
  4280. mminit_verify_pageflags_layout();
  4281. setup_nr_node_ids();
  4282. for_each_online_node(nid) {
  4283. pg_data_t *pgdat = NODE_DATA(nid);
  4284. free_area_init_node(nid, NULL,
  4285. find_min_pfn_for_node(nid), NULL);
  4286. /* Any memory on that node */
  4287. if (pgdat->node_present_pages)
  4288. node_set_state(nid, N_HIGH_MEMORY);
  4289. check_for_regular_memory(pgdat);
  4290. }
  4291. }
  4292. static int __init cmdline_parse_core(char *p, unsigned long *core)
  4293. {
  4294. unsigned long long coremem;
  4295. if (!p)
  4296. return -EINVAL;
  4297. coremem = memparse(p, &p);
  4298. *core = coremem >> PAGE_SHIFT;
  4299. /* Paranoid check that UL is enough for the coremem value */
  4300. WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
  4301. return 0;
  4302. }
  4303. /*
  4304. * kernelcore=size sets the amount of memory for use for allocations that
  4305. * cannot be reclaimed or migrated.
  4306. */
  4307. static int __init cmdline_parse_kernelcore(char *p)
  4308. {
  4309. return cmdline_parse_core(p, &required_kernelcore);
  4310. }
  4311. /*
  4312. * movablecore=size sets the amount of memory for use for allocations that
  4313. * can be reclaimed or migrated.
  4314. */
  4315. static int __init cmdline_parse_movablecore(char *p)
  4316. {
  4317. return cmdline_parse_core(p, &required_movablecore);
  4318. }
  4319. early_param("kernelcore", cmdline_parse_kernelcore);
  4320. early_param("movablecore", cmdline_parse_movablecore);
  4321. #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
  4322. /**
  4323. * set_dma_reserve - set the specified number of pages reserved in the first zone
  4324. * @new_dma_reserve: The number of pages to mark reserved
  4325. *
  4326. * The per-cpu batchsize and zone watermarks are determined by present_pages.
  4327. * In the DMA zone, a significant percentage may be consumed by kernel image
  4328. * and other unfreeable allocations which can skew the watermarks badly. This
  4329. * function may optionally be used to account for unfreeable pages in the
  4330. * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
  4331. * smaller per-cpu batchsize.
  4332. */
  4333. void __init set_dma_reserve(unsigned long new_dma_reserve)
  4334. {
  4335. dma_reserve = new_dma_reserve;
  4336. }
  4337. void __init free_area_init(unsigned long *zones_size)
  4338. {
  4339. free_area_init_node(0, zones_size,
  4340. __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL);
  4341. }
  4342. static int page_alloc_cpu_notify(struct notifier_block *self,
  4343. unsigned long action, void *hcpu)
  4344. {
  4345. int cpu = (unsigned long)hcpu;
  4346. if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
  4347. lru_add_drain_cpu(cpu);
  4348. drain_pages(cpu);
  4349. /*
  4350. * Spill the event counters of the dead processor
  4351. * into the current processors event counters.
  4352. * This artificially elevates the count of the current
  4353. * processor.
  4354. */
  4355. vm_events_fold_cpu(cpu);
  4356. /*
  4357. * Zero the differential counters of the dead processor
  4358. * so that the vm statistics are consistent.
  4359. *
  4360. * This is only okay since the processor is dead and cannot
  4361. * race with what we are doing.
  4362. */
  4363. refresh_cpu_vm_stats(cpu);
  4364. }
  4365. return NOTIFY_OK;
  4366. }
  4367. void __init page_alloc_init(void)
  4368. {
  4369. hotcpu_notifier(page_alloc_cpu_notify, 0);
  4370. }
  4371. /*
  4372. * calculate_totalreserve_pages - called when sysctl_lower_zone_reserve_ratio
  4373. * or min_free_kbytes changes.
  4374. */
  4375. static void calculate_totalreserve_pages(void)
  4376. {
  4377. struct pglist_data *pgdat;
  4378. unsigned long reserve_pages = 0;
  4379. enum zone_type i, j;
  4380. for_each_online_pgdat(pgdat) {
  4381. for (i = 0; i < MAX_NR_ZONES; i++) {
  4382. struct zone *zone = pgdat->node_zones + i;
  4383. unsigned long max = 0;
  4384. /* Find valid and maximum lowmem_reserve in the zone */
  4385. for (j = i; j < MAX_NR_ZONES; j++) {
  4386. if (zone->lowmem_reserve[j] > max)
  4387. max = zone->lowmem_reserve[j];
  4388. }
  4389. /* we treat the high watermark as reserved pages. */
  4390. max += high_wmark_pages(zone);
  4391. if (max > zone->present_pages)
  4392. max = zone->present_pages;
  4393. reserve_pages += max;
  4394. /*
  4395. * Lowmem reserves are not available to
  4396. * GFP_HIGHUSER page cache allocations and
  4397. * kswapd tries to balance zones to their high
  4398. * watermark. As a result, neither should be
  4399. * regarded as dirtyable memory, to prevent a
  4400. * situation where reclaim has to clean pages
  4401. * in order to balance the zones.
  4402. */
  4403. zone->dirty_balance_reserve = max;
  4404. }
  4405. }
  4406. dirty_balance_reserve = reserve_pages;
  4407. totalreserve_pages = reserve_pages;
  4408. }
  4409. /*
  4410. * setup_per_zone_lowmem_reserve - called whenever
  4411. * sysctl_lower_zone_reserve_ratio changes. Ensures that each zone
  4412. * has a correct pages reserved value, so an adequate number of
  4413. * pages are left in the zone after a successful __alloc_pages().
  4414. */
  4415. static void setup_per_zone_lowmem_reserve(void)
  4416. {
  4417. struct pglist_data *pgdat;
  4418. enum zone_type j, idx;
  4419. for_each_online_pgdat(pgdat) {
  4420. for (j = 0; j < MAX_NR_ZONES; j++) {
  4421. struct zone *zone = pgdat->node_zones + j;
  4422. unsigned long present_pages = zone->present_pages;
  4423. zone->lowmem_reserve[j] = 0;
  4424. idx = j;
  4425. while (idx) {
  4426. struct zone *lower_zone;
  4427. idx--;
  4428. if (sysctl_lowmem_reserve_ratio[idx] < 1)
  4429. sysctl_lowmem_reserve_ratio[idx] = 1;
  4430. lower_zone = pgdat->node_zones + idx;
  4431. lower_zone->lowmem_reserve[j] = present_pages /
  4432. sysctl_lowmem_reserve_ratio[idx];
  4433. present_pages += lower_zone->present_pages;
  4434. }
  4435. }
  4436. }
  4437. /* update totalreserve_pages */
  4438. calculate_totalreserve_pages();
  4439. }
  4440. static void __setup_per_zone_wmarks(void)
  4441. {
  4442. unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
  4443. unsigned long lowmem_pages = 0;
  4444. struct zone *zone;
  4445. unsigned long flags;
  4446. /* Calculate total number of !ZONE_HIGHMEM pages */
  4447. for_each_zone(zone) {
  4448. if (!is_highmem(zone))
  4449. lowmem_pages += zone->present_pages;
  4450. }
  4451. for_each_zone(zone) {
  4452. u64 tmp;
  4453. spin_lock_irqsave(&zone->lock, flags);
  4454. tmp = (u64)pages_min * zone->present_pages;
  4455. do_div(tmp, lowmem_pages);
  4456. if (is_highmem(zone)) {
  4457. /*
  4458. * __GFP_HIGH and PF_MEMALLOC allocations usually don't
  4459. * need highmem pages, so cap pages_min to a small
  4460. * value here.
  4461. *
  4462. * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
  4463. * deltas controls asynch page reclaim, and so should
  4464. * not be capped for highmem.
  4465. */
  4466. int min_pages;
  4467. min_pages = zone->present_pages / 1024;
  4468. if (min_pages < SWAP_CLUSTER_MAX)
  4469. min_pages = SWAP_CLUSTER_MAX;
  4470. if (min_pages > 128)
  4471. min_pages = 128;
  4472. zone->watermark[WMARK_MIN] = min_pages;
  4473. } else {
  4474. /*
  4475. * If it's a lowmem zone, reserve a number of pages
  4476. * proportionate to the zone's size.
  4477. */
  4478. zone->watermark[WMARK_MIN] = tmp;
  4479. }
  4480. zone->watermark[WMARK_LOW] = min_wmark_pages(zone) + (tmp >> 2);
  4481. zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + (tmp >> 1);
  4482. zone->watermark[WMARK_MIN] += cma_wmark_pages(zone);
  4483. zone->watermark[WMARK_LOW] += cma_wmark_pages(zone);
  4484. zone->watermark[WMARK_HIGH] += cma_wmark_pages(zone);
  4485. setup_zone_migrate_reserve(zone);
  4486. spin_unlock_irqrestore(&zone->lock, flags);
  4487. }
  4488. /* update totalreserve_pages */
  4489. calculate_totalreserve_pages();
  4490. }
  4491. /**
  4492. * setup_per_zone_wmarks - called when min_free_kbytes changes
  4493. * or when memory is hot-{added|removed}
  4494. *
  4495. * Ensures that the watermark[min,low,high] values for each zone are set
  4496. * correctly with respect to min_free_kbytes.
  4497. */
  4498. void setup_per_zone_wmarks(void)
  4499. {
  4500. mutex_lock(&zonelists_mutex);
  4501. __setup_per_zone_wmarks();
  4502. mutex_unlock(&zonelists_mutex);
  4503. }
  4504. /*
  4505. * The inactive anon list should be small enough that the VM never has to
  4506. * do too much work, but large enough that each inactive page has a chance
  4507. * to be referenced again before it is swapped out.
  4508. *
  4509. * The inactive_anon ratio is the target ratio of ACTIVE_ANON to
  4510. * INACTIVE_ANON pages on this zone's LRU, maintained by the
  4511. * pageout code. A zone->inactive_ratio of 3 means 3:1 or 25% of
  4512. * the anonymous pages are kept on the inactive list.
  4513. *
  4514. * total target max
  4515. * memory ratio inactive anon
  4516. * -------------------------------------
  4517. * 10MB 1 5MB
  4518. * 100MB 1 50MB
  4519. * 1GB 3 250MB
  4520. * 10GB 10 0.9GB
  4521. * 100GB 31 3GB
  4522. * 1TB 101 10GB
  4523. * 10TB 320 32GB
  4524. */
  4525. static void __meminit calculate_zone_inactive_ratio(struct zone *zone)
  4526. {
  4527. unsigned int gb, ratio;
  4528. /* Zone size in gigabytes */
  4529. gb = zone->present_pages >> (30 - PAGE_SHIFT);
  4530. if (gb)
  4531. ratio = int_sqrt(10 * gb);
  4532. else
  4533. ratio = 1;
  4534. zone->inactive_ratio = ratio;
  4535. }
  4536. static void __meminit setup_per_zone_inactive_ratio(void)
  4537. {
  4538. struct zone *zone;
  4539. for_each_zone(zone)
  4540. calculate_zone_inactive_ratio(zone);
  4541. }
  4542. /*
  4543. * Initialise min_free_kbytes.
  4544. *
  4545. * For small machines we want it small (128k min). For large machines
  4546. * we want it large (64MB max). But it is not linear, because network
  4547. * bandwidth does not increase linearly with machine size. We use
  4548. *
  4549. * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy:
  4550. * min_free_kbytes = sqrt(lowmem_kbytes * 16)
  4551. *
  4552. * which yields
  4553. *
  4554. * 16MB: 512k
  4555. * 32MB: 724k
  4556. * 64MB: 1024k
  4557. * 128MB: 1448k
  4558. * 256MB: 2048k
  4559. * 512MB: 2896k
  4560. * 1024MB: 4096k
  4561. * 2048MB: 5792k
  4562. * 4096MB: 8192k
  4563. * 8192MB: 11584k
  4564. * 16384MB: 16384k
  4565. */
  4566. int __meminit init_per_zone_wmark_min(void)
  4567. {
  4568. unsigned long lowmem_kbytes;
  4569. lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
  4570. min_free_kbytes = int_sqrt(lowmem_kbytes * 16);
  4571. if (min_free_kbytes < 128)
  4572. min_free_kbytes = 128;
  4573. if (min_free_kbytes > 65536)
  4574. min_free_kbytes = 65536;
  4575. setup_per_zone_wmarks();
  4576. refresh_zone_stat_thresholds();
  4577. setup_per_zone_lowmem_reserve();
  4578. setup_per_zone_inactive_ratio();
  4579. return 0;
  4580. }
  4581. module_init(init_per_zone_wmark_min)
  4582. /*
  4583. * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so
  4584. * that we can call two helper functions whenever min_free_kbytes
  4585. * changes.
  4586. */
  4587. int min_free_kbytes_sysctl_handler(ctl_table *table, int write,
  4588. void __user *buffer, size_t *length, loff_t *ppos)
  4589. {
  4590. proc_dointvec(table, write, buffer, length, ppos);
  4591. if (write)
  4592. setup_per_zone_wmarks();
  4593. return 0;
  4594. }
  4595. #ifdef CONFIG_NUMA
  4596. int sysctl_min_unmapped_ratio_sysctl_handler(ctl_table *table, int write,
  4597. void __user *buffer, size_t *length, loff_t *ppos)
  4598. {
  4599. struct zone *zone;
  4600. int rc;
  4601. rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
  4602. if (rc)
  4603. return rc;
  4604. for_each_zone(zone)
  4605. zone->min_unmapped_pages = (zone->present_pages *
  4606. sysctl_min_unmapped_ratio) / 100;
  4607. return 0;
  4608. }
  4609. int sysctl_min_slab_ratio_sysctl_handler(ctl_table *table, int write,
  4610. void __user *buffer, size_t *length, loff_t *ppos)
  4611. {
  4612. struct zone *zone;
  4613. int rc;
  4614. rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
  4615. if (rc)
  4616. return rc;
  4617. for_each_zone(zone)
  4618. zone->min_slab_pages = (zone->present_pages *
  4619. sysctl_min_slab_ratio) / 100;
  4620. return 0;
  4621. }
  4622. #endif
  4623. /*
  4624. * lowmem_reserve_ratio_sysctl_handler - just a wrapper around
  4625. * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve()
  4626. * whenever sysctl_lowmem_reserve_ratio changes.
  4627. *
  4628. * The reserve ratio obviously has absolutely no relation with the
  4629. * minimum watermarks. The lowmem reserve ratio can only make sense
  4630. * if in function of the boot time zone sizes.
  4631. */
  4632. int lowmem_reserve_ratio_sysctl_handler(ctl_table *table, int write,
  4633. void __user *buffer, size_t *length, loff_t *ppos)
  4634. {
  4635. proc_dointvec_minmax(table, write, buffer, length, ppos);
  4636. setup_per_zone_lowmem_reserve();
  4637. return 0;
  4638. }
  4639. /*
  4640. * percpu_pagelist_fraction - changes the pcp->high for each zone on each
  4641. * cpu. It is the fraction of total pages in each zone that a hot per cpu pagelist
  4642. * can have before it gets flushed back to buddy allocator.
  4643. */
  4644. int percpu_pagelist_fraction_sysctl_handler(ctl_table *table, int write,
  4645. void __user *buffer, size_t *length, loff_t *ppos)
  4646. {
  4647. struct zone *zone;
  4648. unsigned int cpu;
  4649. int ret;
  4650. ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
  4651. if (!write || (ret < 0))
  4652. return ret;
  4653. for_each_populated_zone(zone) {
  4654. for_each_possible_cpu(cpu) {
  4655. unsigned long high;
  4656. high = zone->present_pages / percpu_pagelist_fraction;
  4657. setup_pagelist_highmark(
  4658. per_cpu_ptr(zone->pageset, cpu), high);
  4659. }
  4660. }
  4661. return 0;
  4662. }
  4663. int hashdist = HASHDIST_DEFAULT;
  4664. #ifdef CONFIG_NUMA
  4665. static int __init set_hashdist(char *str)
  4666. {
  4667. if (!str)
  4668. return 0;
  4669. hashdist = simple_strtoul(str, &str, 0);
  4670. return 1;
  4671. }
  4672. __setup("hashdist=", set_hashdist);
  4673. #endif
  4674. /*
  4675. * allocate a large system hash table from bootmem
  4676. * - it is assumed that the hash table must contain an exact power-of-2
  4677. * quantity of entries
  4678. * - limit is the number of hash buckets, not the total allocation size
  4679. */
  4680. void *__init alloc_large_system_hash(const char *tablename,
  4681. unsigned long bucketsize,
  4682. unsigned long numentries,
  4683. int scale,
  4684. int flags,
  4685. unsigned int *_hash_shift,
  4686. unsigned int *_hash_mask,
  4687. unsigned long low_limit,
  4688. unsigned long high_limit)
  4689. {
  4690. unsigned long long max = high_limit;
  4691. unsigned long log2qty, size;
  4692. void *table = NULL;
  4693. /* allow the kernel cmdline to have a say */
  4694. if (!numentries) {
  4695. /* round applicable memory size up to nearest megabyte */
  4696. numentries = nr_kernel_pages;
  4697. numentries += (1UL << (20 - PAGE_SHIFT)) - 1;
  4698. numentries >>= 20 - PAGE_SHIFT;
  4699. numentries <<= 20 - PAGE_SHIFT;
  4700. /* limit to 1 bucket per 2^scale bytes of low memory */
  4701. if (scale > PAGE_SHIFT)
  4702. numentries >>= (scale - PAGE_SHIFT);
  4703. else
  4704. numentries <<= (PAGE_SHIFT - scale);
  4705. /* Make sure we've got at least a 0-order allocation.. */
  4706. if (unlikely(flags & HASH_SMALL)) {
  4707. /* Makes no sense without HASH_EARLY */
  4708. WARN_ON(!(flags & HASH_EARLY));
  4709. if (!(numentries >> *_hash_shift)) {
  4710. numentries = 1UL << *_hash_shift;
  4711. BUG_ON(!numentries);
  4712. }
  4713. } else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
  4714. numentries = PAGE_SIZE / bucketsize;
  4715. }
  4716. numentries = roundup_pow_of_two(numentries);
  4717. /* limit allocation size to 1/16 total memory by default */
  4718. if (max == 0) {
  4719. max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
  4720. do_div(max, bucketsize);
  4721. }
  4722. max = min(max, 0x80000000ULL);
  4723. if (numentries < low_limit)
  4724. numentries = low_limit;
  4725. if (numentries > max)
  4726. numentries = max;
  4727. log2qty = ilog2(numentries);
  4728. do {
  4729. size = bucketsize << log2qty;
  4730. if (flags & HASH_EARLY)
  4731. table = alloc_bootmem_nopanic(size);
  4732. else if (hashdist)
  4733. table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL);
  4734. else {
  4735. /*
  4736. * If bucketsize is not a power-of-two, we may free
  4737. * some pages at the end of hash table which
  4738. * alloc_pages_exact() automatically does
  4739. */
  4740. if (get_order(size) < MAX_ORDER) {
  4741. table = alloc_pages_exact(size, GFP_ATOMIC);
  4742. kmemleak_alloc(table, size, 1, GFP_ATOMIC);
  4743. }
  4744. }
  4745. } while (!table && size > PAGE_SIZE && --log2qty);
  4746. if (!table)
  4747. panic("Failed to allocate %s hash table\n", tablename);
  4748. printk(KERN_INFO "%s hash table entries: %ld (order: %d, %lu bytes)\n",
  4749. tablename,
  4750. (1UL << log2qty),
  4751. ilog2(size) - PAGE_SHIFT,
  4752. size);
  4753. if (_hash_shift)
  4754. *_hash_shift = log2qty;
  4755. if (_hash_mask)
  4756. *_hash_mask = (1 << log2qty) - 1;
  4757. return table;
  4758. }
  4759. /* Return a pointer to the bitmap storing bits affecting a block of pages */
  4760. static inline unsigned long *get_pageblock_bitmap(struct zone *zone,
  4761. unsigned long pfn)
  4762. {
  4763. #ifdef CONFIG_SPARSEMEM
  4764. return __pfn_to_section(pfn)->pageblock_flags;
  4765. #else
  4766. return zone->pageblock_flags;
  4767. #endif /* CONFIG_SPARSEMEM */
  4768. }
  4769. static inline int pfn_to_bitidx(struct zone *zone, unsigned long pfn)
  4770. {
  4771. #ifdef CONFIG_SPARSEMEM
  4772. pfn &= (PAGES_PER_SECTION-1);
  4773. return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
  4774. #else
  4775. pfn = pfn - zone->zone_start_pfn;
  4776. return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
  4777. #endif /* CONFIG_SPARSEMEM */
  4778. }
  4779. /**
  4780. * get_pageblock_flags_group - Return the requested group of flags for the pageblock_nr_pages block of pages
  4781. * @page: The page within the block of interest
  4782. * @start_bitidx: The first bit of interest to retrieve
  4783. * @end_bitidx: The last bit of interest
  4784. * returns pageblock_bits flags
  4785. */
  4786. unsigned long get_pageblock_flags_group(struct page *page,
  4787. int start_bitidx, int end_bitidx)
  4788. {
  4789. struct zone *zone;
  4790. unsigned long *bitmap;
  4791. unsigned long pfn, bitidx;
  4792. unsigned long flags = 0;
  4793. unsigned long value = 1;
  4794. zone = page_zone(page);
  4795. pfn = page_to_pfn(page);
  4796. bitmap = get_pageblock_bitmap(zone, pfn);
  4797. bitidx = pfn_to_bitidx(zone, pfn);
  4798. for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1)
  4799. if (test_bit(bitidx + start_bitidx, bitmap))
  4800. flags |= value;
  4801. return flags;
  4802. }
  4803. /**
  4804. * set_pageblock_flags_group - Set the requested group of flags for a pageblock_nr_pages block of pages
  4805. * @page: The page within the block of interest
  4806. * @start_bitidx: The first bit of interest
  4807. * @end_bitidx: The last bit of interest
  4808. * @flags: The flags to set
  4809. */
  4810. void set_pageblock_flags_group(struct page *page, unsigned long flags,
  4811. int start_bitidx, int end_bitidx)
  4812. {
  4813. struct zone *zone;
  4814. unsigned long *bitmap;
  4815. unsigned long pfn, bitidx;
  4816. unsigned long value = 1;
  4817. zone = page_zone(page);
  4818. pfn = page_to_pfn(page);
  4819. bitmap = get_pageblock_bitmap(zone, pfn);
  4820. bitidx = pfn_to_bitidx(zone, pfn);
  4821. VM_BUG_ON(pfn < zone->zone_start_pfn);
  4822. VM_BUG_ON(pfn >= zone->zone_start_pfn + zone->spanned_pages);
  4823. for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1)
  4824. if (flags & value)
  4825. __set_bit(bitidx + start_bitidx, bitmap);
  4826. else
  4827. __clear_bit(bitidx + start_bitidx, bitmap);
  4828. }
  4829. /*
  4830. * This function checks whether pageblock includes unmovable pages or not.
  4831. * If @count is not zero, it is okay to include less @count unmovable pages
  4832. *
  4833. * PageLRU check wihtout isolation or lru_lock could race so that
  4834. * MIGRATE_MOVABLE block might include unmovable pages. It means you can't
  4835. * expect this function should be exact.
  4836. */
  4837. bool has_unmovable_pages(struct zone *zone, struct page *page, int count)
  4838. {
  4839. unsigned long pfn, iter, found;
  4840. int mt;
  4841. /*
  4842. * For avoiding noise data, lru_add_drain_all() should be called
  4843. * If ZONE_MOVABLE, the zone never contains unmovable pages
  4844. */
  4845. if (zone_idx(zone) == ZONE_MOVABLE)
  4846. return false;
  4847. mt = get_pageblock_migratetype(page);
  4848. if (mt == MIGRATE_MOVABLE || is_migrate_cma(mt))
  4849. return false;
  4850. pfn = page_to_pfn(page);
  4851. for (found = 0, iter = 0; iter < pageblock_nr_pages; iter++) {
  4852. unsigned long check = pfn + iter;
  4853. if (!pfn_valid_within(check))
  4854. continue;
  4855. page = pfn_to_page(check);
  4856. /*
  4857. * We can't use page_count without pin a page
  4858. * because another CPU can free compound page.
  4859. * This check already skips compound tails of THP
  4860. * because their page->_count is zero at all time.
  4861. */
  4862. if (!atomic_read(&page->_count)) {
  4863. if (PageBuddy(page))
  4864. iter += (1 << page_order(page)) - 1;
  4865. continue;
  4866. }
  4867. if (!PageLRU(page))
  4868. found++;
  4869. /*
  4870. * If there are RECLAIMABLE pages, we need to check it.
  4871. * But now, memory offline itself doesn't call shrink_slab()
  4872. * and it still to be fixed.
  4873. */
  4874. /*
  4875. * If the page is not RAM, page_count()should be 0.
  4876. * we don't need more check. This is an _used_ not-movable page.
  4877. *
  4878. * The problematic thing here is PG_reserved pages. PG_reserved
  4879. * is set to both of a memory hole page and a _used_ kernel
  4880. * page at boot.
  4881. */
  4882. if (found > count)
  4883. return true;
  4884. }
  4885. return false;
  4886. }
  4887. bool is_pageblock_removable_nolock(struct page *page)
  4888. {
  4889. struct zone *zone;
  4890. unsigned long pfn;
  4891. /*
  4892. * We have to be careful here because we are iterating over memory
  4893. * sections which are not zone aware so we might end up outside of
  4894. * the zone but still within the section.
  4895. * We have to take care about the node as well. If the node is offline
  4896. * its NODE_DATA will be NULL - see page_zone.
  4897. */
  4898. if (!node_online(page_to_nid(page)))
  4899. return false;
  4900. zone = page_zone(page);
  4901. pfn = page_to_pfn(page);
  4902. if (zone->zone_start_pfn > pfn ||
  4903. zone->zone_start_pfn + zone->spanned_pages <= pfn)
  4904. return false;
  4905. return !has_unmovable_pages(zone, page, 0);
  4906. }
  4907. #ifdef CONFIG_CMA
  4908. static unsigned long pfn_max_align_down(unsigned long pfn)
  4909. {
  4910. return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES,
  4911. pageblock_nr_pages) - 1);
  4912. }
  4913. static unsigned long pfn_max_align_up(unsigned long pfn)
  4914. {
  4915. return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES,
  4916. pageblock_nr_pages));
  4917. }
  4918. /* [start, end) must belong to a single zone. */
  4919. static int __alloc_contig_migrate_range(struct compact_control *cc,
  4920. unsigned long start, unsigned long end)
  4921. {
  4922. /* This function is based on compact_zone() from compaction.c. */
  4923. unsigned long pfn = start;
  4924. unsigned int tries = 0;
  4925. int ret = 0;
  4926. migrate_prep_local();
  4927. while (pfn < end || !list_empty(&cc->migratepages)) {
  4928. if (fatal_signal_pending(current)) {
  4929. ret = -EINTR;
  4930. break;
  4931. }
  4932. if (list_empty(&cc->migratepages)) {
  4933. cc->nr_migratepages = 0;
  4934. pfn = isolate_migratepages_range(cc->zone, cc,
  4935. pfn, end);
  4936. if (!pfn) {
  4937. ret = -EINTR;
  4938. break;
  4939. }
  4940. tries = 0;
  4941. } else if (++tries == 5) {
  4942. ret = ret < 0 ? ret : -EBUSY;
  4943. break;
  4944. }
  4945. reclaim_clean_pages_from_list(cc->zone, &cc->migratepages);
  4946. ret = migrate_pages(&cc->migratepages,
  4947. alloc_migrate_target,
  4948. 0, false, MIGRATE_SYNC);
  4949. }
  4950. putback_lru_pages(&cc->migratepages);
  4951. return ret > 0 ? 0 : ret;
  4952. }
  4953. /*
  4954. * Update zone's cma pages counter used for watermark level calculation.
  4955. */
  4956. static inline void __update_cma_watermarks(struct zone *zone, int count)
  4957. {
  4958. unsigned long flags;
  4959. spin_lock_irqsave(&zone->lock, flags);
  4960. zone->min_cma_pages += count;
  4961. spin_unlock_irqrestore(&zone->lock, flags);
  4962. setup_per_zone_wmarks();
  4963. }
  4964. /*
  4965. * Trigger memory pressure bump to reclaim some pages in order to be able to
  4966. * allocate 'count' pages in single page units. Does similar work as
  4967. *__alloc_pages_slowpath() function.
  4968. */
  4969. static int __reclaim_pages(struct zone *zone, gfp_t gfp_mask, int count)
  4970. {
  4971. enum zone_type high_zoneidx = gfp_zone(gfp_mask);
  4972. struct zonelist *zonelist = node_zonelist(0, gfp_mask);
  4973. int did_some_progress = 0;
  4974. int order = 1;
  4975. /*
  4976. * Increase level of watermarks to force kswapd do his job
  4977. * to stabilise at new watermark level.
  4978. */
  4979. __update_cma_watermarks(zone, count);
  4980. /* Obey watermarks as if the page was being allocated */
  4981. while (!zone_watermark_ok(zone, 0, low_wmark_pages(zone), 0, 0)) {
  4982. wake_all_kswapd(order, zonelist, high_zoneidx, zone_idx(zone));
  4983. did_some_progress = __perform_reclaim(gfp_mask, order, zonelist,
  4984. NULL);
  4985. if (!did_some_progress) {
  4986. /* Exhausted what can be done so it's blamo time */
  4987. out_of_memory(zonelist, gfp_mask, order, NULL, false);
  4988. }
  4989. }
  4990. /* Restore original watermark levels. */
  4991. __update_cma_watermarks(zone, -count);
  4992. return count;
  4993. }
  4994. /**
  4995. * alloc_contig_range() -- tries to allocate given range of pages
  4996. * @start: start PFN to allocate
  4997. * @end: one-past-the-last PFN to allocate
  4998. * @migratetype: migratetype of the underlaying pageblocks (either
  4999. * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks
  5000. * in range must have the same migratetype and it must
  5001. * be either of the two.
  5002. *
  5003. * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES
  5004. * aligned, however it's the caller's responsibility to guarantee that
  5005. * we are the only thread that changes migrate type of pageblocks the
  5006. * pages fall in.
  5007. *
  5008. * The PFN range must belong to a single zone.
  5009. *
  5010. * Returns zero on success or negative error code. On success all
  5011. * pages which PFN is in [start, end) are allocated for the caller and
  5012. * need to be freed with free_contig_range().
  5013. */
  5014. int alloc_contig_range(unsigned long start, unsigned long end,
  5015. unsigned migratetype)
  5016. {
  5017. struct zone *zone = page_zone(pfn_to_page(start));
  5018. unsigned long outer_start, outer_end;
  5019. int ret = 0, order;
  5020. struct compact_control cc = {
  5021. .nr_migratepages = 0,
  5022. .order = -1,
  5023. .zone = page_zone(pfn_to_page(start)),
  5024. .sync = true,
  5025. .ignore_skip_hint = true,
  5026. };
  5027. INIT_LIST_HEAD(&cc.migratepages);
  5028. /*
  5029. * What we do here is we mark all pageblocks in range as
  5030. * MIGRATE_ISOLATE. Because pageblock and max order pages may
  5031. * have different sizes, and due to the way page allocator
  5032. * work, we align the range to biggest of the two pages so
  5033. * that page allocator won't try to merge buddies from
  5034. * different pageblocks and change MIGRATE_ISOLATE to some
  5035. * other migration type.
  5036. *
  5037. * Once the pageblocks are marked as MIGRATE_ISOLATE, we
  5038. * migrate the pages from an unaligned range (ie. pages that
  5039. * we are interested in). This will put all the pages in
  5040. * range back to page allocator as MIGRATE_ISOLATE.
  5041. *
  5042. * When this is done, we take the pages in range from page
  5043. * allocator removing them from the buddy system. This way
  5044. * page allocator will never consider using them.
  5045. *
  5046. * This lets us mark the pageblocks back as
  5047. * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the
  5048. * aligned range but not in the unaligned, original range are
  5049. * put back to page allocator so that buddy can use them.
  5050. */
  5051. ret = start_isolate_page_range(pfn_max_align_down(start),
  5052. pfn_max_align_up(end), migratetype);
  5053. if (ret)
  5054. goto done;
  5055. ret = __alloc_contig_migrate_range(&cc, start, end);
  5056. if (ret)
  5057. goto done;
  5058. /*
  5059. * Pages from [start, end) are within a MAX_ORDER_NR_PAGES
  5060. * aligned blocks that are marked as MIGRATE_ISOLATE. What's
  5061. * more, all pages in [start, end) are free in page allocator.
  5062. * What we are going to do is to allocate all pages from
  5063. * [start, end) (that is remove them from page allocator).
  5064. *
  5065. * The only problem is that pages at the beginning and at the
  5066. * end of interesting range may be not aligned with pages that
  5067. * page allocator holds, ie. they can be part of higher order
  5068. * pages. Because of this, we reserve the bigger range and
  5069. * once this is done free the pages we are not interested in.
  5070. *
  5071. * We don't have to hold zone->lock here because the pages are
  5072. * isolated thus they won't get removed from buddy.
  5073. */
  5074. lru_add_drain_all();
  5075. drain_all_pages();
  5076. order = 0;
  5077. outer_start = start;
  5078. while (!PageBuddy(pfn_to_page(outer_start))) {
  5079. if (++order >= MAX_ORDER) {
  5080. ret = -EBUSY;
  5081. goto done;
  5082. }
  5083. outer_start &= ~0UL << order;
  5084. }
  5085. /* Make sure the range is really isolated. */
  5086. if (test_pages_isolated(outer_start, end)) {
  5087. pr_warn("alloc_contig_range test_pages_isolated(%lx, %lx) failed\n",
  5088. outer_start, end);
  5089. ret = -EBUSY;
  5090. goto done;
  5091. }
  5092. /*
  5093. * Reclaim enough pages to make sure that contiguous allocation
  5094. * will not starve the system.
  5095. */
  5096. __reclaim_pages(zone, GFP_HIGHUSER_MOVABLE, end-start);
  5097. /* Grab isolated pages from freelists. */
  5098. outer_end = isolate_freepages_range(&cc, outer_start, end);
  5099. if (!outer_end) {
  5100. ret = -EBUSY;
  5101. goto done;
  5102. }
  5103. /* Free head and tail (if any) */
  5104. if (start != outer_start)
  5105. free_contig_range(outer_start, start - outer_start);
  5106. if (end != outer_end)
  5107. free_contig_range(end, outer_end - end);
  5108. done:
  5109. undo_isolate_page_range(pfn_max_align_down(start),
  5110. pfn_max_align_up(end), migratetype);
  5111. return ret;
  5112. }
  5113. void free_contig_range(unsigned long pfn, unsigned nr_pages)
  5114. {
  5115. for (; nr_pages--; ++pfn)
  5116. __free_page(pfn_to_page(pfn));
  5117. }
  5118. #endif
  5119. #ifdef CONFIG_MEMORY_HOTPLUG
  5120. static int __meminit __zone_pcp_update(void *data)
  5121. {
  5122. struct zone *zone = data;
  5123. int cpu;
  5124. unsigned long batch = zone_batchsize(zone), flags;
  5125. for_each_possible_cpu(cpu) {
  5126. struct per_cpu_pageset *pset;
  5127. struct per_cpu_pages *pcp;
  5128. pset = per_cpu_ptr(zone->pageset, cpu);
  5129. pcp = &pset->pcp;
  5130. local_irq_save(flags);
  5131. if (pcp->count > 0)
  5132. free_pcppages_bulk(zone, pcp->count, pcp);
  5133. setup_pageset(pset, batch);
  5134. local_irq_restore(flags);
  5135. }
  5136. return 0;
  5137. }
  5138. void __meminit zone_pcp_update(struct zone *zone)
  5139. {
  5140. stop_machine(__zone_pcp_update, zone, NULL);
  5141. }
  5142. #endif
  5143. #ifdef CONFIG_MEMORY_HOTREMOVE
  5144. void zone_pcp_reset(struct zone *zone)
  5145. {
  5146. unsigned long flags;
  5147. /* avoid races with drain_pages() */
  5148. local_irq_save(flags);
  5149. if (zone->pageset != &boot_pageset) {
  5150. free_percpu(zone->pageset);
  5151. zone->pageset = &boot_pageset;
  5152. }
  5153. local_irq_restore(flags);
  5154. }
  5155. /*
  5156. * All pages in the range must be isolated before calling this.
  5157. */
  5158. void
  5159. __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
  5160. {
  5161. struct page *page;
  5162. struct zone *zone;
  5163. int order, i;
  5164. unsigned long pfn;
  5165. unsigned long flags;
  5166. /* find the first valid pfn */
  5167. for (pfn = start_pfn; pfn < end_pfn; pfn++)
  5168. if (pfn_valid(pfn))
  5169. break;
  5170. if (pfn == end_pfn)
  5171. return;
  5172. zone = page_zone(pfn_to_page(pfn));
  5173. spin_lock_irqsave(&zone->lock, flags);
  5174. pfn = start_pfn;
  5175. while (pfn < end_pfn) {
  5176. if (!pfn_valid(pfn)) {
  5177. pfn++;
  5178. continue;
  5179. }
  5180. page = pfn_to_page(pfn);
  5181. BUG_ON(page_count(page));
  5182. BUG_ON(!PageBuddy(page));
  5183. order = page_order(page);
  5184. #ifdef CONFIG_DEBUG_VM
  5185. printk(KERN_INFO "remove from free list %lx %d %lx\n",
  5186. pfn, 1 << order, end_pfn);
  5187. #endif
  5188. list_del(&page->lru);
  5189. rmv_page_order(page);
  5190. zone->free_area[order].nr_free--;
  5191. __mod_zone_page_state(zone, NR_FREE_PAGES,
  5192. - (1UL << order));
  5193. for (i = 0; i < (1 << order); i++)
  5194. SetPageReserved((page+i));
  5195. pfn += (1 << order);
  5196. }
  5197. spin_unlock_irqrestore(&zone->lock, flags);
  5198. }
  5199. #endif
  5200. #ifdef CONFIG_MEMORY_FAILURE
  5201. bool is_free_buddy_page(struct page *page)
  5202. {
  5203. struct zone *zone = page_zone(page);
  5204. unsigned long pfn = page_to_pfn(page);
  5205. unsigned long flags;
  5206. int order;
  5207. spin_lock_irqsave(&zone->lock, flags);
  5208. for (order = 0; order < MAX_ORDER; order++) {
  5209. struct page *page_head = page - (pfn & ((1 << order) - 1));
  5210. if (PageBuddy(page_head) && page_order(page_head) >= order)
  5211. break;
  5212. }
  5213. spin_unlock_irqrestore(&zone->lock, flags);
  5214. return order < MAX_ORDER;
  5215. }
  5216. #endif
  5217. static const struct trace_print_flags pageflag_names[] = {
  5218. {1UL << PG_locked, "locked" },
  5219. {1UL << PG_error, "error" },
  5220. {1UL << PG_referenced, "referenced" },
  5221. {1UL << PG_uptodate, "uptodate" },
  5222. {1UL << PG_dirty, "dirty" },
  5223. {1UL << PG_lru, "lru" },
  5224. {1UL << PG_active, "active" },
  5225. {1UL << PG_slab, "slab" },
  5226. {1UL << PG_owner_priv_1, "owner_priv_1" },
  5227. {1UL << PG_arch_1, "arch_1" },
  5228. {1UL << PG_reserved, "reserved" },
  5229. {1UL << PG_private, "private" },
  5230. {1UL << PG_private_2, "private_2" },
  5231. {1UL << PG_writeback, "writeback" },
  5232. #ifdef CONFIG_PAGEFLAGS_EXTENDED
  5233. {1UL << PG_head, "head" },
  5234. {1UL << PG_tail, "tail" },
  5235. #else
  5236. {1UL << PG_compound, "compound" },
  5237. #endif
  5238. {1UL << PG_swapcache, "swapcache" },
  5239. {1UL << PG_mappedtodisk, "mappedtodisk" },
  5240. {1UL << PG_reclaim, "reclaim" },
  5241. {1UL << PG_swapbacked, "swapbacked" },
  5242. {1UL << PG_unevictable, "unevictable" },
  5243. #ifdef CONFIG_MMU
  5244. {1UL << PG_mlocked, "mlocked" },
  5245. #endif
  5246. #ifdef CONFIG_ARCH_USES_PG_UNCACHED
  5247. {1UL << PG_uncached, "uncached" },
  5248. #endif
  5249. #ifdef CONFIG_MEMORY_FAILURE
  5250. {1UL << PG_hwpoison, "hwpoison" },
  5251. #endif
  5252. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  5253. {1UL << PG_compound_lock, "compound_lock" },
  5254. #endif
  5255. };
  5256. static void dump_page_flags(unsigned long flags)
  5257. {
  5258. const char *delim = "";
  5259. unsigned long mask;
  5260. int i;
  5261. BUILD_BUG_ON(ARRAY_SIZE(pageflag_names) != __NR_PAGEFLAGS);
  5262. printk(KERN_ALERT "page flags: %#lx(", flags);
  5263. /* remove zone id */
  5264. flags &= (1UL << NR_PAGEFLAGS) - 1;
  5265. for (i = 0; i < ARRAY_SIZE(pageflag_names) && flags; i++) {
  5266. mask = pageflag_names[i].mask;
  5267. if ((flags & mask) != mask)
  5268. continue;
  5269. flags &= ~mask;
  5270. printk("%s%s", delim, pageflag_names[i].name);
  5271. delim = "|";
  5272. }
  5273. /* check for left over flags */
  5274. if (flags)
  5275. printk("%s%#lx", delim, flags);
  5276. printk(")\n");
  5277. }
  5278. void dump_page(struct page *page)
  5279. {
  5280. printk(KERN_ALERT
  5281. "page:%p count:%d mapcount:%d mapping:%p index:%#lx\n",
  5282. page, atomic_read(&page->_count), page_mapcount(page),
  5283. page->mapping, page->index);
  5284. dump_page_flags(page->flags);
  5285. mem_cgroup_print_bad_page(page);
  5286. }
  5287. /* reset zone->present_pages */
  5288. void reset_zone_present_pages(void)
  5289. {
  5290. struct zone *z;
  5291. int i, nid;
  5292. for_each_node_state(nid, N_HIGH_MEMORY) {
  5293. for (i = 0; i < MAX_NR_ZONES; i++) {
  5294. z = NODE_DATA(nid)->node_zones + i;
  5295. z->present_pages = 0;
  5296. }
  5297. }
  5298. }
  5299. /* calculate zone's present pages in buddy system */
  5300. void fixup_zone_present_pages(int nid, unsigned long start_pfn,
  5301. unsigned long end_pfn)
  5302. {
  5303. struct zone *z;
  5304. unsigned long zone_start_pfn, zone_end_pfn;
  5305. int i;
  5306. for (i = 0; i < MAX_NR_ZONES; i++) {
  5307. z = NODE_DATA(nid)->node_zones + i;
  5308. zone_start_pfn = z->zone_start_pfn;
  5309. zone_end_pfn = zone_start_pfn + z->spanned_pages;
  5310. /* if the two regions intersect */
  5311. if (!(zone_start_pfn >= end_pfn || zone_end_pfn <= start_pfn))
  5312. z->present_pages += min(end_pfn, zone_end_pfn) -
  5313. max(start_pfn, zone_start_pfn);
  5314. }
  5315. }