page_alloc.c 160 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809
  1. /*
  2. * linux/mm/page_alloc.c
  3. *
  4. * Manages the free list, the system allocates free pages here.
  5. * Note that kmalloc() lives in slab.c
  6. *
  7. * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
  8. * Swap reorganised 29.12.95, Stephen Tweedie
  9. * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
  10. * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
  11. * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
  12. * Zone balancing, Kanoj Sarcar, SGI, Jan 2000
  13. * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
  14. * (lots of bits borrowed from Ingo Molnar & Andrew Morton)
  15. */
  16. #include <linux/stddef.h>
  17. #include <linux/mm.h>
  18. #include <linux/swap.h>
  19. #include <linux/interrupt.h>
  20. #include <linux/pagemap.h>
  21. #include <linux/jiffies.h>
  22. #include <linux/bootmem.h>
  23. #include <linux/memblock.h>
  24. #include <linux/compiler.h>
  25. #include <linux/kernel.h>
  26. #include <linux/kmemcheck.h>
  27. #include <linux/module.h>
  28. #include <linux/suspend.h>
  29. #include <linux/pagevec.h>
  30. #include <linux/blkdev.h>
  31. #include <linux/slab.h>
  32. #include <linux/ratelimit.h>
  33. #include <linux/oom.h>
  34. #include <linux/notifier.h>
  35. #include <linux/topology.h>
  36. #include <linux/sysctl.h>
  37. #include <linux/cpu.h>
  38. #include <linux/cpuset.h>
  39. #include <linux/memory_hotplug.h>
  40. #include <linux/nodemask.h>
  41. #include <linux/vmalloc.h>
  42. #include <linux/vmstat.h>
  43. #include <linux/mempolicy.h>
  44. #include <linux/stop_machine.h>
  45. #include <linux/sort.h>
  46. #include <linux/pfn.h>
  47. #include <linux/backing-dev.h>
  48. #include <linux/fault-inject.h>
  49. #include <linux/page-isolation.h>
  50. #include <linux/page_cgroup.h>
  51. #include <linux/debugobjects.h>
  52. #include <linux/kmemleak.h>
  53. #include <linux/memory.h>
  54. #include <linux/compaction.h>
  55. #include <trace/events/kmem.h>
  56. #include <linux/ftrace_event.h>
  57. #include <linux/memcontrol.h>
  58. #include <linux/prefetch.h>
  59. #include <asm/tlbflush.h>
  60. #include <asm/div64.h>
  61. #include "internal.h"
  62. #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
  63. DEFINE_PER_CPU(int, numa_node);
  64. EXPORT_PER_CPU_SYMBOL(numa_node);
  65. #endif
  66. #ifdef CONFIG_HAVE_MEMORYLESS_NODES
  67. /*
  68. * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
  69. * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
  70. * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
  71. * defined in <linux/topology.h>.
  72. */
  73. DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */
  74. EXPORT_PER_CPU_SYMBOL(_numa_mem_);
  75. #endif
  76. /*
  77. * Array of node states.
  78. */
  79. nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
  80. [N_POSSIBLE] = NODE_MASK_ALL,
  81. [N_ONLINE] = { { [0] = 1UL } },
  82. #ifndef CONFIG_NUMA
  83. [N_NORMAL_MEMORY] = { { [0] = 1UL } },
  84. #ifdef CONFIG_HIGHMEM
  85. [N_HIGH_MEMORY] = { { [0] = 1UL } },
  86. #endif
  87. [N_CPU] = { { [0] = 1UL } },
  88. #endif /* NUMA */
  89. };
  90. EXPORT_SYMBOL(node_states);
  91. unsigned long totalram_pages __read_mostly;
  92. unsigned long totalreserve_pages __read_mostly;
  93. int percpu_pagelist_fraction;
  94. gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
  95. #ifdef CONFIG_PM_SLEEP
  96. /*
  97. * The following functions are used by the suspend/hibernate code to temporarily
  98. * change gfp_allowed_mask in order to avoid using I/O during memory allocations
  99. * while devices are suspended. To avoid races with the suspend/hibernate code,
  100. * they should always be called with pm_mutex held (gfp_allowed_mask also should
  101. * only be modified with pm_mutex held, unless the suspend/hibernate code is
  102. * guaranteed not to run in parallel with that modification).
  103. */
  104. static gfp_t saved_gfp_mask;
  105. void pm_restore_gfp_mask(void)
  106. {
  107. WARN_ON(!mutex_is_locked(&pm_mutex));
  108. if (saved_gfp_mask) {
  109. gfp_allowed_mask = saved_gfp_mask;
  110. saved_gfp_mask = 0;
  111. }
  112. }
  113. void pm_restrict_gfp_mask(void)
  114. {
  115. WARN_ON(!mutex_is_locked(&pm_mutex));
  116. WARN_ON(saved_gfp_mask);
  117. saved_gfp_mask = gfp_allowed_mask;
  118. gfp_allowed_mask &= ~GFP_IOFS;
  119. }
  120. #endif /* CONFIG_PM_SLEEP */
  121. #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
  122. int pageblock_order __read_mostly;
  123. #endif
  124. static void __free_pages_ok(struct page *page, unsigned int order);
  125. /*
  126. * results with 256, 32 in the lowmem_reserve sysctl:
  127. * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
  128. * 1G machine -> (16M dma, 784M normal, 224M high)
  129. * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
  130. * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
  131. * HIGHMEM allocation will (224M+784M)/256 of ram reserved in ZONE_DMA
  132. *
  133. * TBD: should special case ZONE_DMA32 machines here - in those we normally
  134. * don't need any ZONE_NORMAL reservation
  135. */
  136. int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = {
  137. #ifdef CONFIG_ZONE_DMA
  138. 256,
  139. #endif
  140. #ifdef CONFIG_ZONE_DMA32
  141. 256,
  142. #endif
  143. #ifdef CONFIG_HIGHMEM
  144. 32,
  145. #endif
  146. 32,
  147. };
  148. EXPORT_SYMBOL(totalram_pages);
  149. static char * const zone_names[MAX_NR_ZONES] = {
  150. #ifdef CONFIG_ZONE_DMA
  151. "DMA",
  152. #endif
  153. #ifdef CONFIG_ZONE_DMA32
  154. "DMA32",
  155. #endif
  156. "Normal",
  157. #ifdef CONFIG_HIGHMEM
  158. "HighMem",
  159. #endif
  160. "Movable",
  161. };
  162. int min_free_kbytes = 1024;
  163. static unsigned long __meminitdata nr_kernel_pages;
  164. static unsigned long __meminitdata nr_all_pages;
  165. static unsigned long __meminitdata dma_reserve;
  166. #ifdef CONFIG_ARCH_POPULATES_NODE_MAP
  167. /*
  168. * MAX_ACTIVE_REGIONS determines the maximum number of distinct
  169. * ranges of memory (RAM) that may be registered with add_active_range().
  170. * Ranges passed to add_active_range() will be merged if possible
  171. * so the number of times add_active_range() can be called is
  172. * related to the number of nodes and the number of holes
  173. */
  174. #ifdef CONFIG_MAX_ACTIVE_REGIONS
  175. /* Allow an architecture to set MAX_ACTIVE_REGIONS to save memory */
  176. #define MAX_ACTIVE_REGIONS CONFIG_MAX_ACTIVE_REGIONS
  177. #else
  178. #if MAX_NUMNODES >= 32
  179. /* If there can be many nodes, allow up to 50 holes per node */
  180. #define MAX_ACTIVE_REGIONS (MAX_NUMNODES*50)
  181. #else
  182. /* By default, allow up to 256 distinct regions */
  183. #define MAX_ACTIVE_REGIONS 256
  184. #endif
  185. #endif
  186. static struct node_active_region __meminitdata early_node_map[MAX_ACTIVE_REGIONS];
  187. static int __meminitdata nr_nodemap_entries;
  188. static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES];
  189. static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES];
  190. static unsigned long __initdata required_kernelcore;
  191. static unsigned long __initdata required_movablecore;
  192. static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES];
  193. /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
  194. int movable_zone;
  195. EXPORT_SYMBOL(movable_zone);
  196. #endif /* CONFIG_ARCH_POPULATES_NODE_MAP */
  197. #if MAX_NUMNODES > 1
  198. int nr_node_ids __read_mostly = MAX_NUMNODES;
  199. int nr_online_nodes __read_mostly = 1;
  200. EXPORT_SYMBOL(nr_node_ids);
  201. EXPORT_SYMBOL(nr_online_nodes);
  202. #endif
  203. int page_group_by_mobility_disabled __read_mostly;
  204. static void set_pageblock_migratetype(struct page *page, int migratetype)
  205. {
  206. if (unlikely(page_group_by_mobility_disabled))
  207. migratetype = MIGRATE_UNMOVABLE;
  208. set_pageblock_flags_group(page, (unsigned long)migratetype,
  209. PB_migrate, PB_migrate_end);
  210. }
  211. bool oom_killer_disabled __read_mostly;
  212. #ifdef CONFIG_DEBUG_VM
  213. static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
  214. {
  215. int ret = 0;
  216. unsigned seq;
  217. unsigned long pfn = page_to_pfn(page);
  218. do {
  219. seq = zone_span_seqbegin(zone);
  220. if (pfn >= zone->zone_start_pfn + zone->spanned_pages)
  221. ret = 1;
  222. else if (pfn < zone->zone_start_pfn)
  223. ret = 1;
  224. } while (zone_span_seqretry(zone, seq));
  225. return ret;
  226. }
  227. static int page_is_consistent(struct zone *zone, struct page *page)
  228. {
  229. if (!pfn_valid_within(page_to_pfn(page)))
  230. return 0;
  231. if (zone != page_zone(page))
  232. return 0;
  233. return 1;
  234. }
  235. /*
  236. * Temporary debugging check for pages not lying within a given zone.
  237. */
  238. static int bad_range(struct zone *zone, struct page *page)
  239. {
  240. if (page_outside_zone_boundaries(zone, page))
  241. return 1;
  242. if (!page_is_consistent(zone, page))
  243. return 1;
  244. return 0;
  245. }
  246. #else
  247. static inline int bad_range(struct zone *zone, struct page *page)
  248. {
  249. return 0;
  250. }
  251. #endif
  252. static void bad_page(struct page *page)
  253. {
  254. static unsigned long resume;
  255. static unsigned long nr_shown;
  256. static unsigned long nr_unshown;
  257. /* Don't complain about poisoned pages */
  258. if (PageHWPoison(page)) {
  259. reset_page_mapcount(page); /* remove PageBuddy */
  260. return;
  261. }
  262. /*
  263. * Allow a burst of 60 reports, then keep quiet for that minute;
  264. * or allow a steady drip of one report per second.
  265. */
  266. if (nr_shown == 60) {
  267. if (time_before(jiffies, resume)) {
  268. nr_unshown++;
  269. goto out;
  270. }
  271. if (nr_unshown) {
  272. printk(KERN_ALERT
  273. "BUG: Bad page state: %lu messages suppressed\n",
  274. nr_unshown);
  275. nr_unshown = 0;
  276. }
  277. nr_shown = 0;
  278. }
  279. if (nr_shown++ == 0)
  280. resume = jiffies + 60 * HZ;
  281. printk(KERN_ALERT "BUG: Bad page state in process %s pfn:%05lx\n",
  282. current->comm, page_to_pfn(page));
  283. dump_page(page);
  284. dump_stack();
  285. out:
  286. /* Leave bad fields for debug, except PageBuddy could make trouble */
  287. reset_page_mapcount(page); /* remove PageBuddy */
  288. add_taint(TAINT_BAD_PAGE);
  289. }
  290. /*
  291. * Higher-order pages are called "compound pages". They are structured thusly:
  292. *
  293. * The first PAGE_SIZE page is called the "head page".
  294. *
  295. * The remaining PAGE_SIZE pages are called "tail pages".
  296. *
  297. * All pages have PG_compound set. All pages have their ->private pointing at
  298. * the head page (even the head page has this).
  299. *
  300. * The first tail page's ->lru.next holds the address of the compound page's
  301. * put_page() function. Its ->lru.prev holds the order of allocation.
  302. * This usage means that zero-order pages may not be compound.
  303. */
  304. static void free_compound_page(struct page *page)
  305. {
  306. __free_pages_ok(page, compound_order(page));
  307. }
  308. void prep_compound_page(struct page *page, unsigned long order)
  309. {
  310. int i;
  311. int nr_pages = 1 << order;
  312. set_compound_page_dtor(page, free_compound_page);
  313. set_compound_order(page, order);
  314. __SetPageHead(page);
  315. for (i = 1; i < nr_pages; i++) {
  316. struct page *p = page + i;
  317. __SetPageTail(p);
  318. p->first_page = page;
  319. }
  320. }
  321. /* update __split_huge_page_refcount if you change this function */
  322. static int destroy_compound_page(struct page *page, unsigned long order)
  323. {
  324. int i;
  325. int nr_pages = 1 << order;
  326. int bad = 0;
  327. if (unlikely(compound_order(page) != order) ||
  328. unlikely(!PageHead(page))) {
  329. bad_page(page);
  330. bad++;
  331. }
  332. __ClearPageHead(page);
  333. for (i = 1; i < nr_pages; i++) {
  334. struct page *p = page + i;
  335. if (unlikely(!PageTail(p) || (p->first_page != page))) {
  336. bad_page(page);
  337. bad++;
  338. }
  339. __ClearPageTail(p);
  340. }
  341. return bad;
  342. }
  343. static inline void prep_zero_page(struct page *page, int order, gfp_t gfp_flags)
  344. {
  345. int i;
  346. /*
  347. * clear_highpage() will use KM_USER0, so it's a bug to use __GFP_ZERO
  348. * and __GFP_HIGHMEM from hard or soft interrupt context.
  349. */
  350. VM_BUG_ON((gfp_flags & __GFP_HIGHMEM) && in_interrupt());
  351. for (i = 0; i < (1 << order); i++)
  352. clear_highpage(page + i);
  353. }
  354. static inline void set_page_order(struct page *page, int order)
  355. {
  356. set_page_private(page, order);
  357. __SetPageBuddy(page);
  358. }
  359. static inline void rmv_page_order(struct page *page)
  360. {
  361. __ClearPageBuddy(page);
  362. set_page_private(page, 0);
  363. }
  364. /*
  365. * Locate the struct page for both the matching buddy in our
  366. * pair (buddy1) and the combined O(n+1) page they form (page).
  367. *
  368. * 1) Any buddy B1 will have an order O twin B2 which satisfies
  369. * the following equation:
  370. * B2 = B1 ^ (1 << O)
  371. * For example, if the starting buddy (buddy2) is #8 its order
  372. * 1 buddy is #10:
  373. * B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10
  374. *
  375. * 2) Any buddy B will have an order O+1 parent P which
  376. * satisfies the following equation:
  377. * P = B & ~(1 << O)
  378. *
  379. * Assumption: *_mem_map is contiguous at least up to MAX_ORDER
  380. */
  381. static inline unsigned long
  382. __find_buddy_index(unsigned long page_idx, unsigned int order)
  383. {
  384. return page_idx ^ (1 << order);
  385. }
  386. /*
  387. * This function checks whether a page is free && is the buddy
  388. * we can do coalesce a page and its buddy if
  389. * (a) the buddy is not in a hole &&
  390. * (b) the buddy is in the buddy system &&
  391. * (c) a page and its buddy have the same order &&
  392. * (d) a page and its buddy are in the same zone.
  393. *
  394. * For recording whether a page is in the buddy system, we set ->_mapcount -2.
  395. * Setting, clearing, and testing _mapcount -2 is serialized by zone->lock.
  396. *
  397. * For recording page's order, we use page_private(page).
  398. */
  399. static inline int page_is_buddy(struct page *page, struct page *buddy,
  400. int order)
  401. {
  402. if (!pfn_valid_within(page_to_pfn(buddy)))
  403. return 0;
  404. if (page_zone_id(page) != page_zone_id(buddy))
  405. return 0;
  406. if (PageBuddy(buddy) && page_order(buddy) == order) {
  407. VM_BUG_ON(page_count(buddy) != 0);
  408. return 1;
  409. }
  410. return 0;
  411. }
  412. /*
  413. * Freeing function for a buddy system allocator.
  414. *
  415. * The concept of a buddy system is to maintain direct-mapped table
  416. * (containing bit values) for memory blocks of various "orders".
  417. * The bottom level table contains the map for the smallest allocatable
  418. * units of memory (here, pages), and each level above it describes
  419. * pairs of units from the levels below, hence, "buddies".
  420. * At a high level, all that happens here is marking the table entry
  421. * at the bottom level available, and propagating the changes upward
  422. * as necessary, plus some accounting needed to play nicely with other
  423. * parts of the VM system.
  424. * At each level, we keep a list of pages, which are heads of continuous
  425. * free pages of length of (1 << order) and marked with _mapcount -2. Page's
  426. * order is recorded in page_private(page) field.
  427. * So when we are allocating or freeing one, we can derive the state of the
  428. * other. That is, if we allocate a small block, and both were
  429. * free, the remainder of the region must be split into blocks.
  430. * If a block is freed, and its buddy is also free, then this
  431. * triggers coalescing into a block of larger size.
  432. *
  433. * -- wli
  434. */
  435. static inline void __free_one_page(struct page *page,
  436. struct zone *zone, unsigned int order,
  437. int migratetype)
  438. {
  439. unsigned long page_idx;
  440. unsigned long combined_idx;
  441. unsigned long uninitialized_var(buddy_idx);
  442. struct page *buddy;
  443. if (unlikely(PageCompound(page)))
  444. if (unlikely(destroy_compound_page(page, order)))
  445. return;
  446. VM_BUG_ON(migratetype == -1);
  447. page_idx = page_to_pfn(page) & ((1 << MAX_ORDER) - 1);
  448. VM_BUG_ON(page_idx & ((1 << order) - 1));
  449. VM_BUG_ON(bad_range(zone, page));
  450. while (order < MAX_ORDER-1) {
  451. buddy_idx = __find_buddy_index(page_idx, order);
  452. buddy = page + (buddy_idx - page_idx);
  453. if (!page_is_buddy(page, buddy, order))
  454. break;
  455. /* Our buddy is free, merge with it and move up one order. */
  456. list_del(&buddy->lru);
  457. zone->free_area[order].nr_free--;
  458. rmv_page_order(buddy);
  459. combined_idx = buddy_idx & page_idx;
  460. page = page + (combined_idx - page_idx);
  461. page_idx = combined_idx;
  462. order++;
  463. }
  464. set_page_order(page, order);
  465. /*
  466. * If this is not the largest possible page, check if the buddy
  467. * of the next-highest order is free. If it is, it's possible
  468. * that pages are being freed that will coalesce soon. In case,
  469. * that is happening, add the free page to the tail of the list
  470. * so it's less likely to be used soon and more likely to be merged
  471. * as a higher order page
  472. */
  473. if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {
  474. struct page *higher_page, *higher_buddy;
  475. combined_idx = buddy_idx & page_idx;
  476. higher_page = page + (combined_idx - page_idx);
  477. buddy_idx = __find_buddy_index(combined_idx, order + 1);
  478. higher_buddy = page + (buddy_idx - combined_idx);
  479. if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
  480. list_add_tail(&page->lru,
  481. &zone->free_area[order].free_list[migratetype]);
  482. goto out;
  483. }
  484. }
  485. list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
  486. out:
  487. zone->free_area[order].nr_free++;
  488. }
  489. /*
  490. * free_page_mlock() -- clean up attempts to free and mlocked() page.
  491. * Page should not be on lru, so no need to fix that up.
  492. * free_pages_check() will verify...
  493. */
  494. static inline void free_page_mlock(struct page *page)
  495. {
  496. __dec_zone_page_state(page, NR_MLOCK);
  497. __count_vm_event(UNEVICTABLE_MLOCKFREED);
  498. }
  499. static inline int free_pages_check(struct page *page)
  500. {
  501. if (unlikely(page_mapcount(page) |
  502. (page->mapping != NULL) |
  503. (atomic_read(&page->_count) != 0) |
  504. (page->flags & PAGE_FLAGS_CHECK_AT_FREE) |
  505. (mem_cgroup_bad_page_check(page)))) {
  506. bad_page(page);
  507. return 1;
  508. }
  509. if (page->flags & PAGE_FLAGS_CHECK_AT_PREP)
  510. page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
  511. return 0;
  512. }
  513. /*
  514. * Frees a number of pages from the PCP lists
  515. * Assumes all pages on list are in same zone, and of same order.
  516. * count is the number of pages to free.
  517. *
  518. * If the zone was previously in an "all pages pinned" state then look to
  519. * see if this freeing clears that state.
  520. *
  521. * And clear the zone's pages_scanned counter, to hold off the "all pages are
  522. * pinned" detection logic.
  523. */
  524. static void free_pcppages_bulk(struct zone *zone, int count,
  525. struct per_cpu_pages *pcp)
  526. {
  527. int migratetype = 0;
  528. int batch_free = 0;
  529. int to_free = count;
  530. spin_lock(&zone->lock);
  531. zone->all_unreclaimable = 0;
  532. zone->pages_scanned = 0;
  533. while (to_free) {
  534. struct page *page;
  535. struct list_head *list;
  536. /*
  537. * Remove pages from lists in a round-robin fashion. A
  538. * batch_free count is maintained that is incremented when an
  539. * empty list is encountered. This is so more pages are freed
  540. * off fuller lists instead of spinning excessively around empty
  541. * lists
  542. */
  543. do {
  544. batch_free++;
  545. if (++migratetype == MIGRATE_PCPTYPES)
  546. migratetype = 0;
  547. list = &pcp->lists[migratetype];
  548. } while (list_empty(list));
  549. /* This is the only non-empty list. Free them all. */
  550. if (batch_free == MIGRATE_PCPTYPES)
  551. batch_free = to_free;
  552. do {
  553. page = list_entry(list->prev, struct page, lru);
  554. /* must delete as __free_one_page list manipulates */
  555. list_del(&page->lru);
  556. /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */
  557. __free_one_page(page, zone, 0, page_private(page));
  558. trace_mm_page_pcpu_drain(page, 0, page_private(page));
  559. } while (--to_free && --batch_free && !list_empty(list));
  560. }
  561. __mod_zone_page_state(zone, NR_FREE_PAGES, count);
  562. spin_unlock(&zone->lock);
  563. }
  564. static void free_one_page(struct zone *zone, struct page *page, int order,
  565. int migratetype)
  566. {
  567. spin_lock(&zone->lock);
  568. zone->all_unreclaimable = 0;
  569. zone->pages_scanned = 0;
  570. __free_one_page(page, zone, order, migratetype);
  571. __mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order);
  572. spin_unlock(&zone->lock);
  573. }
  574. static bool free_pages_prepare(struct page *page, unsigned int order)
  575. {
  576. int i;
  577. int bad = 0;
  578. trace_mm_page_free_direct(page, order);
  579. kmemcheck_free_shadow(page, order);
  580. if (PageAnon(page))
  581. page->mapping = NULL;
  582. for (i = 0; i < (1 << order); i++)
  583. bad += free_pages_check(page + i);
  584. if (bad)
  585. return false;
  586. if (!PageHighMem(page)) {
  587. debug_check_no_locks_freed(page_address(page),PAGE_SIZE<<order);
  588. debug_check_no_obj_freed(page_address(page),
  589. PAGE_SIZE << order);
  590. }
  591. arch_free_page(page, order);
  592. kernel_map_pages(page, 1 << order, 0);
  593. return true;
  594. }
  595. static void __free_pages_ok(struct page *page, unsigned int order)
  596. {
  597. unsigned long flags;
  598. int wasMlocked = __TestClearPageMlocked(page);
  599. if (!free_pages_prepare(page, order))
  600. return;
  601. local_irq_save(flags);
  602. if (unlikely(wasMlocked))
  603. free_page_mlock(page);
  604. __count_vm_events(PGFREE, 1 << order);
  605. free_one_page(page_zone(page), page, order,
  606. get_pageblock_migratetype(page));
  607. local_irq_restore(flags);
  608. }
  609. /*
  610. * permit the bootmem allocator to evade page validation on high-order frees
  611. */
  612. void __meminit __free_pages_bootmem(struct page *page, unsigned int order)
  613. {
  614. if (order == 0) {
  615. __ClearPageReserved(page);
  616. set_page_count(page, 0);
  617. set_page_refcounted(page);
  618. __free_page(page);
  619. } else {
  620. int loop;
  621. prefetchw(page);
  622. for (loop = 0; loop < BITS_PER_LONG; loop++) {
  623. struct page *p = &page[loop];
  624. if (loop + 1 < BITS_PER_LONG)
  625. prefetchw(p + 1);
  626. __ClearPageReserved(p);
  627. set_page_count(p, 0);
  628. }
  629. set_page_refcounted(page);
  630. __free_pages(page, order);
  631. }
  632. }
  633. /*
  634. * The order of subdivision here is critical for the IO subsystem.
  635. * Please do not alter this order without good reasons and regression
  636. * testing. Specifically, as large blocks of memory are subdivided,
  637. * the order in which smaller blocks are delivered depends on the order
  638. * they're subdivided in this function. This is the primary factor
  639. * influencing the order in which pages are delivered to the IO
  640. * subsystem according to empirical testing, and this is also justified
  641. * by considering the behavior of a buddy system containing a single
  642. * large block of memory acted on by a series of small allocations.
  643. * This behavior is a critical factor in sglist merging's success.
  644. *
  645. * -- wli
  646. */
  647. static inline void expand(struct zone *zone, struct page *page,
  648. int low, int high, struct free_area *area,
  649. int migratetype)
  650. {
  651. unsigned long size = 1 << high;
  652. while (high > low) {
  653. area--;
  654. high--;
  655. size >>= 1;
  656. VM_BUG_ON(bad_range(zone, &page[size]));
  657. list_add(&page[size].lru, &area->free_list[migratetype]);
  658. area->nr_free++;
  659. set_page_order(&page[size], high);
  660. }
  661. }
  662. /*
  663. * This page is about to be returned from the page allocator
  664. */
  665. static inline int check_new_page(struct page *page)
  666. {
  667. if (unlikely(page_mapcount(page) |
  668. (page->mapping != NULL) |
  669. (atomic_read(&page->_count) != 0) |
  670. (page->flags & PAGE_FLAGS_CHECK_AT_PREP) |
  671. (mem_cgroup_bad_page_check(page)))) {
  672. bad_page(page);
  673. return 1;
  674. }
  675. return 0;
  676. }
  677. static int prep_new_page(struct page *page, int order, gfp_t gfp_flags)
  678. {
  679. int i;
  680. for (i = 0; i < (1 << order); i++) {
  681. struct page *p = page + i;
  682. if (unlikely(check_new_page(p)))
  683. return 1;
  684. }
  685. set_page_private(page, 0);
  686. set_page_refcounted(page);
  687. arch_alloc_page(page, order);
  688. kernel_map_pages(page, 1 << order, 1);
  689. if (gfp_flags & __GFP_ZERO)
  690. prep_zero_page(page, order, gfp_flags);
  691. if (order && (gfp_flags & __GFP_COMP))
  692. prep_compound_page(page, order);
  693. return 0;
  694. }
  695. /*
  696. * Go through the free lists for the given migratetype and remove
  697. * the smallest available page from the freelists
  698. */
  699. static inline
  700. struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
  701. int migratetype)
  702. {
  703. unsigned int current_order;
  704. struct free_area * area;
  705. struct page *page;
  706. /* Find a page of the appropriate size in the preferred list */
  707. for (current_order = order; current_order < MAX_ORDER; ++current_order) {
  708. area = &(zone->free_area[current_order]);
  709. if (list_empty(&area->free_list[migratetype]))
  710. continue;
  711. page = list_entry(area->free_list[migratetype].next,
  712. struct page, lru);
  713. list_del(&page->lru);
  714. rmv_page_order(page);
  715. area->nr_free--;
  716. expand(zone, page, order, current_order, area, migratetype);
  717. return page;
  718. }
  719. return NULL;
  720. }
  721. /*
  722. * This array describes the order lists are fallen back to when
  723. * the free lists for the desirable migrate type are depleted
  724. */
  725. static int fallbacks[MIGRATE_TYPES][MIGRATE_TYPES-1] = {
  726. [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE },
  727. [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE },
  728. [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },
  729. [MIGRATE_RESERVE] = { MIGRATE_RESERVE, MIGRATE_RESERVE, MIGRATE_RESERVE }, /* Never used */
  730. };
  731. /*
  732. * Move the free pages in a range to the free lists of the requested type.
  733. * Note that start_page and end_pages are not aligned on a pageblock
  734. * boundary. If alignment is required, use move_freepages_block()
  735. */
  736. static int move_freepages(struct zone *zone,
  737. struct page *start_page, struct page *end_page,
  738. int migratetype)
  739. {
  740. struct page *page;
  741. unsigned long order;
  742. int pages_moved = 0;
  743. #ifndef CONFIG_HOLES_IN_ZONE
  744. /*
  745. * page_zone is not safe to call in this context when
  746. * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant
  747. * anyway as we check zone boundaries in move_freepages_block().
  748. * Remove at a later date when no bug reports exist related to
  749. * grouping pages by mobility
  750. */
  751. BUG_ON(page_zone(start_page) != page_zone(end_page));
  752. #endif
  753. for (page = start_page; page <= end_page;) {
  754. /* Make sure we are not inadvertently changing nodes */
  755. VM_BUG_ON(page_to_nid(page) != zone_to_nid(zone));
  756. if (!pfn_valid_within(page_to_pfn(page))) {
  757. page++;
  758. continue;
  759. }
  760. if (!PageBuddy(page)) {
  761. page++;
  762. continue;
  763. }
  764. order = page_order(page);
  765. list_move(&page->lru,
  766. &zone->free_area[order].free_list[migratetype]);
  767. page += 1 << order;
  768. pages_moved += 1 << order;
  769. }
  770. return pages_moved;
  771. }
  772. static int move_freepages_block(struct zone *zone, struct page *page,
  773. int migratetype)
  774. {
  775. unsigned long start_pfn, end_pfn;
  776. struct page *start_page, *end_page;
  777. start_pfn = page_to_pfn(page);
  778. start_pfn = start_pfn & ~(pageblock_nr_pages-1);
  779. start_page = pfn_to_page(start_pfn);
  780. end_page = start_page + pageblock_nr_pages - 1;
  781. end_pfn = start_pfn + pageblock_nr_pages - 1;
  782. /* Do not cross zone boundaries */
  783. if (start_pfn < zone->zone_start_pfn)
  784. start_page = page;
  785. if (end_pfn >= zone->zone_start_pfn + zone->spanned_pages)
  786. return 0;
  787. return move_freepages(zone, start_page, end_page, migratetype);
  788. }
  789. static void change_pageblock_range(struct page *pageblock_page,
  790. int start_order, int migratetype)
  791. {
  792. int nr_pageblocks = 1 << (start_order - pageblock_order);
  793. while (nr_pageblocks--) {
  794. set_pageblock_migratetype(pageblock_page, migratetype);
  795. pageblock_page += pageblock_nr_pages;
  796. }
  797. }
  798. /* Remove an element from the buddy allocator from the fallback list */
  799. static inline struct page *
  800. __rmqueue_fallback(struct zone *zone, int order, int start_migratetype)
  801. {
  802. struct free_area * area;
  803. int current_order;
  804. struct page *page;
  805. int migratetype, i;
  806. /* Find the largest possible block of pages in the other list */
  807. for (current_order = MAX_ORDER-1; current_order >= order;
  808. --current_order) {
  809. for (i = 0; i < MIGRATE_TYPES - 1; i++) {
  810. migratetype = fallbacks[start_migratetype][i];
  811. /* MIGRATE_RESERVE handled later if necessary */
  812. if (migratetype == MIGRATE_RESERVE)
  813. continue;
  814. area = &(zone->free_area[current_order]);
  815. if (list_empty(&area->free_list[migratetype]))
  816. continue;
  817. page = list_entry(area->free_list[migratetype].next,
  818. struct page, lru);
  819. area->nr_free--;
  820. /*
  821. * If breaking a large block of pages, move all free
  822. * pages to the preferred allocation list. If falling
  823. * back for a reclaimable kernel allocation, be more
  824. * aggressive about taking ownership of free pages
  825. */
  826. if (unlikely(current_order >= (pageblock_order >> 1)) ||
  827. start_migratetype == MIGRATE_RECLAIMABLE ||
  828. page_group_by_mobility_disabled) {
  829. unsigned long pages;
  830. pages = move_freepages_block(zone, page,
  831. start_migratetype);
  832. /* Claim the whole block if over half of it is free */
  833. if (pages >= (1 << (pageblock_order-1)) ||
  834. page_group_by_mobility_disabled)
  835. set_pageblock_migratetype(page,
  836. start_migratetype);
  837. migratetype = start_migratetype;
  838. }
  839. /* Remove the page from the freelists */
  840. list_del(&page->lru);
  841. rmv_page_order(page);
  842. /* Take ownership for orders >= pageblock_order */
  843. if (current_order >= pageblock_order)
  844. change_pageblock_range(page, current_order,
  845. start_migratetype);
  846. expand(zone, page, order, current_order, area, migratetype);
  847. trace_mm_page_alloc_extfrag(page, order, current_order,
  848. start_migratetype, migratetype);
  849. return page;
  850. }
  851. }
  852. return NULL;
  853. }
  854. /*
  855. * Do the hard work of removing an element from the buddy allocator.
  856. * Call me with the zone->lock already held.
  857. */
  858. static struct page *__rmqueue(struct zone *zone, unsigned int order,
  859. int migratetype)
  860. {
  861. struct page *page;
  862. retry_reserve:
  863. page = __rmqueue_smallest(zone, order, migratetype);
  864. if (unlikely(!page) && migratetype != MIGRATE_RESERVE) {
  865. page = __rmqueue_fallback(zone, order, migratetype);
  866. /*
  867. * Use MIGRATE_RESERVE rather than fail an allocation. goto
  868. * is used because __rmqueue_smallest is an inline function
  869. * and we want just one call site
  870. */
  871. if (!page) {
  872. migratetype = MIGRATE_RESERVE;
  873. goto retry_reserve;
  874. }
  875. }
  876. trace_mm_page_alloc_zone_locked(page, order, migratetype);
  877. return page;
  878. }
  879. /*
  880. * Obtain a specified number of elements from the buddy allocator, all under
  881. * a single hold of the lock, for efficiency. Add them to the supplied list.
  882. * Returns the number of new pages which were placed at *list.
  883. */
  884. static int rmqueue_bulk(struct zone *zone, unsigned int order,
  885. unsigned long count, struct list_head *list,
  886. int migratetype, int cold)
  887. {
  888. int i;
  889. spin_lock(&zone->lock);
  890. for (i = 0; i < count; ++i) {
  891. struct page *page = __rmqueue(zone, order, migratetype);
  892. if (unlikely(page == NULL))
  893. break;
  894. /*
  895. * Split buddy pages returned by expand() are received here
  896. * in physical page order. The page is added to the callers and
  897. * list and the list head then moves forward. From the callers
  898. * perspective, the linked list is ordered by page number in
  899. * some conditions. This is useful for IO devices that can
  900. * merge IO requests if the physical pages are ordered
  901. * properly.
  902. */
  903. if (likely(cold == 0))
  904. list_add(&page->lru, list);
  905. else
  906. list_add_tail(&page->lru, list);
  907. set_page_private(page, migratetype);
  908. list = &page->lru;
  909. }
  910. __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
  911. spin_unlock(&zone->lock);
  912. return i;
  913. }
  914. #ifdef CONFIG_NUMA
  915. /*
  916. * Called from the vmstat counter updater to drain pagesets of this
  917. * currently executing processor on remote nodes after they have
  918. * expired.
  919. *
  920. * Note that this function must be called with the thread pinned to
  921. * a single processor.
  922. */
  923. void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
  924. {
  925. unsigned long flags;
  926. int to_drain;
  927. local_irq_save(flags);
  928. if (pcp->count >= pcp->batch)
  929. to_drain = pcp->batch;
  930. else
  931. to_drain = pcp->count;
  932. free_pcppages_bulk(zone, to_drain, pcp);
  933. pcp->count -= to_drain;
  934. local_irq_restore(flags);
  935. }
  936. #endif
  937. /*
  938. * Drain pages of the indicated processor.
  939. *
  940. * The processor must either be the current processor and the
  941. * thread pinned to the current processor or a processor that
  942. * is not online.
  943. */
  944. static void drain_pages(unsigned int cpu)
  945. {
  946. unsigned long flags;
  947. struct zone *zone;
  948. for_each_populated_zone(zone) {
  949. struct per_cpu_pageset *pset;
  950. struct per_cpu_pages *pcp;
  951. local_irq_save(flags);
  952. pset = per_cpu_ptr(zone->pageset, cpu);
  953. pcp = &pset->pcp;
  954. if (pcp->count) {
  955. free_pcppages_bulk(zone, pcp->count, pcp);
  956. pcp->count = 0;
  957. }
  958. local_irq_restore(flags);
  959. }
  960. }
  961. /*
  962. * Spill all of this CPU's per-cpu pages back into the buddy allocator.
  963. */
  964. void drain_local_pages(void *arg)
  965. {
  966. drain_pages(smp_processor_id());
  967. }
  968. /*
  969. * Spill all the per-cpu pages from all CPUs back into the buddy allocator
  970. */
  971. void drain_all_pages(void)
  972. {
  973. on_each_cpu(drain_local_pages, NULL, 1);
  974. }
  975. #ifdef CONFIG_HIBERNATION
  976. void mark_free_pages(struct zone *zone)
  977. {
  978. unsigned long pfn, max_zone_pfn;
  979. unsigned long flags;
  980. int order, t;
  981. struct list_head *curr;
  982. if (!zone->spanned_pages)
  983. return;
  984. spin_lock_irqsave(&zone->lock, flags);
  985. max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
  986. for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
  987. if (pfn_valid(pfn)) {
  988. struct page *page = pfn_to_page(pfn);
  989. if (!swsusp_page_is_forbidden(page))
  990. swsusp_unset_page_free(page);
  991. }
  992. for_each_migratetype_order(order, t) {
  993. list_for_each(curr, &zone->free_area[order].free_list[t]) {
  994. unsigned long i;
  995. pfn = page_to_pfn(list_entry(curr, struct page, lru));
  996. for (i = 0; i < (1UL << order); i++)
  997. swsusp_set_page_free(pfn_to_page(pfn + i));
  998. }
  999. }
  1000. spin_unlock_irqrestore(&zone->lock, flags);
  1001. }
  1002. #endif /* CONFIG_PM */
  1003. /*
  1004. * Free a 0-order page
  1005. * cold == 1 ? free a cold page : free a hot page
  1006. */
  1007. void free_hot_cold_page(struct page *page, int cold)
  1008. {
  1009. struct zone *zone = page_zone(page);
  1010. struct per_cpu_pages *pcp;
  1011. unsigned long flags;
  1012. int migratetype;
  1013. int wasMlocked = __TestClearPageMlocked(page);
  1014. if (!free_pages_prepare(page, 0))
  1015. return;
  1016. migratetype = get_pageblock_migratetype(page);
  1017. set_page_private(page, migratetype);
  1018. local_irq_save(flags);
  1019. if (unlikely(wasMlocked))
  1020. free_page_mlock(page);
  1021. __count_vm_event(PGFREE);
  1022. /*
  1023. * We only track unmovable, reclaimable and movable on pcp lists.
  1024. * Free ISOLATE pages back to the allocator because they are being
  1025. * offlined but treat RESERVE as movable pages so we can get those
  1026. * areas back if necessary. Otherwise, we may have to free
  1027. * excessively into the page allocator
  1028. */
  1029. if (migratetype >= MIGRATE_PCPTYPES) {
  1030. if (unlikely(migratetype == MIGRATE_ISOLATE)) {
  1031. free_one_page(zone, page, 0, migratetype);
  1032. goto out;
  1033. }
  1034. migratetype = MIGRATE_MOVABLE;
  1035. }
  1036. pcp = &this_cpu_ptr(zone->pageset)->pcp;
  1037. if (cold)
  1038. list_add_tail(&page->lru, &pcp->lists[migratetype]);
  1039. else
  1040. list_add(&page->lru, &pcp->lists[migratetype]);
  1041. pcp->count++;
  1042. if (pcp->count >= pcp->high) {
  1043. free_pcppages_bulk(zone, pcp->batch, pcp);
  1044. pcp->count -= pcp->batch;
  1045. }
  1046. out:
  1047. local_irq_restore(flags);
  1048. }
  1049. /*
  1050. * split_page takes a non-compound higher-order page, and splits it into
  1051. * n (1<<order) sub-pages: page[0..n]
  1052. * Each sub-page must be freed individually.
  1053. *
  1054. * Note: this is probably too low level an operation for use in drivers.
  1055. * Please consult with lkml before using this in your driver.
  1056. */
  1057. void split_page(struct page *page, unsigned int order)
  1058. {
  1059. int i;
  1060. VM_BUG_ON(PageCompound(page));
  1061. VM_BUG_ON(!page_count(page));
  1062. #ifdef CONFIG_KMEMCHECK
  1063. /*
  1064. * Split shadow pages too, because free(page[0]) would
  1065. * otherwise free the whole shadow.
  1066. */
  1067. if (kmemcheck_page_is_tracked(page))
  1068. split_page(virt_to_page(page[0].shadow), order);
  1069. #endif
  1070. for (i = 1; i < (1 << order); i++)
  1071. set_page_refcounted(page + i);
  1072. }
  1073. /*
  1074. * Similar to split_page except the page is already free. As this is only
  1075. * being used for migration, the migratetype of the block also changes.
  1076. * As this is called with interrupts disabled, the caller is responsible
  1077. * for calling arch_alloc_page() and kernel_map_page() after interrupts
  1078. * are enabled.
  1079. *
  1080. * Note: this is probably too low level an operation for use in drivers.
  1081. * Please consult with lkml before using this in your driver.
  1082. */
  1083. int split_free_page(struct page *page)
  1084. {
  1085. unsigned int order;
  1086. unsigned long watermark;
  1087. struct zone *zone;
  1088. BUG_ON(!PageBuddy(page));
  1089. zone = page_zone(page);
  1090. order = page_order(page);
  1091. /* Obey watermarks as if the page was being allocated */
  1092. watermark = low_wmark_pages(zone) + (1 << order);
  1093. if (!zone_watermark_ok(zone, 0, watermark, 0, 0))
  1094. return 0;
  1095. /* Remove page from free list */
  1096. list_del(&page->lru);
  1097. zone->free_area[order].nr_free--;
  1098. rmv_page_order(page);
  1099. __mod_zone_page_state(zone, NR_FREE_PAGES, -(1UL << order));
  1100. /* Split into individual pages */
  1101. set_page_refcounted(page);
  1102. split_page(page, order);
  1103. if (order >= pageblock_order - 1) {
  1104. struct page *endpage = page + (1 << order) - 1;
  1105. for (; page < endpage; page += pageblock_nr_pages)
  1106. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  1107. }
  1108. return 1 << order;
  1109. }
  1110. /*
  1111. * Really, prep_compound_page() should be called from __rmqueue_bulk(). But
  1112. * we cheat by calling it from here, in the order > 0 path. Saves a branch
  1113. * or two.
  1114. */
  1115. static inline
  1116. struct page *buffered_rmqueue(struct zone *preferred_zone,
  1117. struct zone *zone, int order, gfp_t gfp_flags,
  1118. int migratetype)
  1119. {
  1120. unsigned long flags;
  1121. struct page *page;
  1122. int cold = !!(gfp_flags & __GFP_COLD);
  1123. again:
  1124. if (likely(order == 0)) {
  1125. struct per_cpu_pages *pcp;
  1126. struct list_head *list;
  1127. local_irq_save(flags);
  1128. pcp = &this_cpu_ptr(zone->pageset)->pcp;
  1129. list = &pcp->lists[migratetype];
  1130. if (list_empty(list)) {
  1131. pcp->count += rmqueue_bulk(zone, 0,
  1132. pcp->batch, list,
  1133. migratetype, cold);
  1134. if (unlikely(list_empty(list)))
  1135. goto failed;
  1136. }
  1137. if (cold)
  1138. page = list_entry(list->prev, struct page, lru);
  1139. else
  1140. page = list_entry(list->next, struct page, lru);
  1141. list_del(&page->lru);
  1142. pcp->count--;
  1143. } else {
  1144. if (unlikely(gfp_flags & __GFP_NOFAIL)) {
  1145. /*
  1146. * __GFP_NOFAIL is not to be used in new code.
  1147. *
  1148. * All __GFP_NOFAIL callers should be fixed so that they
  1149. * properly detect and handle allocation failures.
  1150. *
  1151. * We most definitely don't want callers attempting to
  1152. * allocate greater than order-1 page units with
  1153. * __GFP_NOFAIL.
  1154. */
  1155. WARN_ON_ONCE(order > 1);
  1156. }
  1157. spin_lock_irqsave(&zone->lock, flags);
  1158. page = __rmqueue(zone, order, migratetype);
  1159. spin_unlock(&zone->lock);
  1160. if (!page)
  1161. goto failed;
  1162. __mod_zone_page_state(zone, NR_FREE_PAGES, -(1 << order));
  1163. }
  1164. __count_zone_vm_events(PGALLOC, zone, 1 << order);
  1165. zone_statistics(preferred_zone, zone, gfp_flags);
  1166. local_irq_restore(flags);
  1167. VM_BUG_ON(bad_range(zone, page));
  1168. if (prep_new_page(page, order, gfp_flags))
  1169. goto again;
  1170. return page;
  1171. failed:
  1172. local_irq_restore(flags);
  1173. return NULL;
  1174. }
  1175. /* The ALLOC_WMARK bits are used as an index to zone->watermark */
  1176. #define ALLOC_WMARK_MIN WMARK_MIN
  1177. #define ALLOC_WMARK_LOW WMARK_LOW
  1178. #define ALLOC_WMARK_HIGH WMARK_HIGH
  1179. #define ALLOC_NO_WATERMARKS 0x04 /* don't check watermarks at all */
  1180. /* Mask to get the watermark bits */
  1181. #define ALLOC_WMARK_MASK (ALLOC_NO_WATERMARKS-1)
  1182. #define ALLOC_HARDER 0x10 /* try to alloc harder */
  1183. #define ALLOC_HIGH 0x20 /* __GFP_HIGH set */
  1184. #define ALLOC_CPUSET 0x40 /* check for correct cpuset */
  1185. #ifdef CONFIG_FAIL_PAGE_ALLOC
  1186. static struct fail_page_alloc_attr {
  1187. struct fault_attr attr;
  1188. u32 ignore_gfp_highmem;
  1189. u32 ignore_gfp_wait;
  1190. u32 min_order;
  1191. #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
  1192. struct dentry *ignore_gfp_highmem_file;
  1193. struct dentry *ignore_gfp_wait_file;
  1194. struct dentry *min_order_file;
  1195. #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
  1196. } fail_page_alloc = {
  1197. .attr = FAULT_ATTR_INITIALIZER,
  1198. .ignore_gfp_wait = 1,
  1199. .ignore_gfp_highmem = 1,
  1200. .min_order = 1,
  1201. };
  1202. static int __init setup_fail_page_alloc(char *str)
  1203. {
  1204. return setup_fault_attr(&fail_page_alloc.attr, str);
  1205. }
  1206. __setup("fail_page_alloc=", setup_fail_page_alloc);
  1207. static int should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
  1208. {
  1209. if (order < fail_page_alloc.min_order)
  1210. return 0;
  1211. if (gfp_mask & __GFP_NOFAIL)
  1212. return 0;
  1213. if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
  1214. return 0;
  1215. if (fail_page_alloc.ignore_gfp_wait && (gfp_mask & __GFP_WAIT))
  1216. return 0;
  1217. return should_fail(&fail_page_alloc.attr, 1 << order);
  1218. }
  1219. #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
  1220. static int __init fail_page_alloc_debugfs(void)
  1221. {
  1222. mode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
  1223. struct dentry *dir;
  1224. int err;
  1225. err = init_fault_attr_dentries(&fail_page_alloc.attr,
  1226. "fail_page_alloc");
  1227. if (err)
  1228. return err;
  1229. dir = fail_page_alloc.attr.dentries.dir;
  1230. fail_page_alloc.ignore_gfp_wait_file =
  1231. debugfs_create_bool("ignore-gfp-wait", mode, dir,
  1232. &fail_page_alloc.ignore_gfp_wait);
  1233. fail_page_alloc.ignore_gfp_highmem_file =
  1234. debugfs_create_bool("ignore-gfp-highmem", mode, dir,
  1235. &fail_page_alloc.ignore_gfp_highmem);
  1236. fail_page_alloc.min_order_file =
  1237. debugfs_create_u32("min-order", mode, dir,
  1238. &fail_page_alloc.min_order);
  1239. if (!fail_page_alloc.ignore_gfp_wait_file ||
  1240. !fail_page_alloc.ignore_gfp_highmem_file ||
  1241. !fail_page_alloc.min_order_file) {
  1242. err = -ENOMEM;
  1243. debugfs_remove(fail_page_alloc.ignore_gfp_wait_file);
  1244. debugfs_remove(fail_page_alloc.ignore_gfp_highmem_file);
  1245. debugfs_remove(fail_page_alloc.min_order_file);
  1246. cleanup_fault_attr_dentries(&fail_page_alloc.attr);
  1247. }
  1248. return err;
  1249. }
  1250. late_initcall(fail_page_alloc_debugfs);
  1251. #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
  1252. #else /* CONFIG_FAIL_PAGE_ALLOC */
  1253. static inline int should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
  1254. {
  1255. return 0;
  1256. }
  1257. #endif /* CONFIG_FAIL_PAGE_ALLOC */
  1258. /*
  1259. * Return true if free pages are above 'mark'. This takes into account the order
  1260. * of the allocation.
  1261. */
  1262. static bool __zone_watermark_ok(struct zone *z, int order, unsigned long mark,
  1263. int classzone_idx, int alloc_flags, long free_pages)
  1264. {
  1265. /* free_pages my go negative - that's OK */
  1266. long min = mark;
  1267. int o;
  1268. free_pages -= (1 << order) + 1;
  1269. if (alloc_flags & ALLOC_HIGH)
  1270. min -= min / 2;
  1271. if (alloc_flags & ALLOC_HARDER)
  1272. min -= min / 4;
  1273. if (free_pages <= min + z->lowmem_reserve[classzone_idx])
  1274. return false;
  1275. for (o = 0; o < order; o++) {
  1276. /* At the next order, this order's pages become unavailable */
  1277. free_pages -= z->free_area[o].nr_free << o;
  1278. /* Require fewer higher order pages to be free */
  1279. min >>= 1;
  1280. if (free_pages <= min)
  1281. return false;
  1282. }
  1283. return true;
  1284. }
  1285. bool zone_watermark_ok(struct zone *z, int order, unsigned long mark,
  1286. int classzone_idx, int alloc_flags)
  1287. {
  1288. return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
  1289. zone_page_state(z, NR_FREE_PAGES));
  1290. }
  1291. bool zone_watermark_ok_safe(struct zone *z, int order, unsigned long mark,
  1292. int classzone_idx, int alloc_flags)
  1293. {
  1294. long free_pages = zone_page_state(z, NR_FREE_PAGES);
  1295. if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
  1296. free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
  1297. return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
  1298. free_pages);
  1299. }
  1300. #ifdef CONFIG_NUMA
  1301. /*
  1302. * zlc_setup - Setup for "zonelist cache". Uses cached zone data to
  1303. * skip over zones that are not allowed by the cpuset, or that have
  1304. * been recently (in last second) found to be nearly full. See further
  1305. * comments in mmzone.h. Reduces cache footprint of zonelist scans
  1306. * that have to skip over a lot of full or unallowed zones.
  1307. *
  1308. * If the zonelist cache is present in the passed in zonelist, then
  1309. * returns a pointer to the allowed node mask (either the current
  1310. * tasks mems_allowed, or node_states[N_HIGH_MEMORY].)
  1311. *
  1312. * If the zonelist cache is not available for this zonelist, does
  1313. * nothing and returns NULL.
  1314. *
  1315. * If the fullzones BITMAP in the zonelist cache is stale (more than
  1316. * a second since last zap'd) then we zap it out (clear its bits.)
  1317. *
  1318. * We hold off even calling zlc_setup, until after we've checked the
  1319. * first zone in the zonelist, on the theory that most allocations will
  1320. * be satisfied from that first zone, so best to examine that zone as
  1321. * quickly as we can.
  1322. */
  1323. static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
  1324. {
  1325. struct zonelist_cache *zlc; /* cached zonelist speedup info */
  1326. nodemask_t *allowednodes; /* zonelist_cache approximation */
  1327. zlc = zonelist->zlcache_ptr;
  1328. if (!zlc)
  1329. return NULL;
  1330. if (time_after(jiffies, zlc->last_full_zap + HZ)) {
  1331. bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
  1332. zlc->last_full_zap = jiffies;
  1333. }
  1334. allowednodes = !in_interrupt() && (alloc_flags & ALLOC_CPUSET) ?
  1335. &cpuset_current_mems_allowed :
  1336. &node_states[N_HIGH_MEMORY];
  1337. return allowednodes;
  1338. }
  1339. /*
  1340. * Given 'z' scanning a zonelist, run a couple of quick checks to see
  1341. * if it is worth looking at further for free memory:
  1342. * 1) Check that the zone isn't thought to be full (doesn't have its
  1343. * bit set in the zonelist_cache fullzones BITMAP).
  1344. * 2) Check that the zones node (obtained from the zonelist_cache
  1345. * z_to_n[] mapping) is allowed in the passed in allowednodes mask.
  1346. * Return true (non-zero) if zone is worth looking at further, or
  1347. * else return false (zero) if it is not.
  1348. *
  1349. * This check -ignores- the distinction between various watermarks,
  1350. * such as GFP_HIGH, GFP_ATOMIC, PF_MEMALLOC, ... If a zone is
  1351. * found to be full for any variation of these watermarks, it will
  1352. * be considered full for up to one second by all requests, unless
  1353. * we are so low on memory on all allowed nodes that we are forced
  1354. * into the second scan of the zonelist.
  1355. *
  1356. * In the second scan we ignore this zonelist cache and exactly
  1357. * apply the watermarks to all zones, even it is slower to do so.
  1358. * We are low on memory in the second scan, and should leave no stone
  1359. * unturned looking for a free page.
  1360. */
  1361. static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z,
  1362. nodemask_t *allowednodes)
  1363. {
  1364. struct zonelist_cache *zlc; /* cached zonelist speedup info */
  1365. int i; /* index of *z in zonelist zones */
  1366. int n; /* node that zone *z is on */
  1367. zlc = zonelist->zlcache_ptr;
  1368. if (!zlc)
  1369. return 1;
  1370. i = z - zonelist->_zonerefs;
  1371. n = zlc->z_to_n[i];
  1372. /* This zone is worth trying if it is allowed but not full */
  1373. return node_isset(n, *allowednodes) && !test_bit(i, zlc->fullzones);
  1374. }
  1375. /*
  1376. * Given 'z' scanning a zonelist, set the corresponding bit in
  1377. * zlc->fullzones, so that subsequent attempts to allocate a page
  1378. * from that zone don't waste time re-examining it.
  1379. */
  1380. static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z)
  1381. {
  1382. struct zonelist_cache *zlc; /* cached zonelist speedup info */
  1383. int i; /* index of *z in zonelist zones */
  1384. zlc = zonelist->zlcache_ptr;
  1385. if (!zlc)
  1386. return;
  1387. i = z - zonelist->_zonerefs;
  1388. set_bit(i, zlc->fullzones);
  1389. }
  1390. #else /* CONFIG_NUMA */
  1391. static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
  1392. {
  1393. return NULL;
  1394. }
  1395. static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z,
  1396. nodemask_t *allowednodes)
  1397. {
  1398. return 1;
  1399. }
  1400. static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z)
  1401. {
  1402. }
  1403. #endif /* CONFIG_NUMA */
  1404. /*
  1405. * get_page_from_freelist goes through the zonelist trying to allocate
  1406. * a page.
  1407. */
  1408. static struct page *
  1409. get_page_from_freelist(gfp_t gfp_mask, nodemask_t *nodemask, unsigned int order,
  1410. struct zonelist *zonelist, int high_zoneidx, int alloc_flags,
  1411. struct zone *preferred_zone, int migratetype)
  1412. {
  1413. struct zoneref *z;
  1414. struct page *page = NULL;
  1415. int classzone_idx;
  1416. struct zone *zone;
  1417. nodemask_t *allowednodes = NULL;/* zonelist_cache approximation */
  1418. int zlc_active = 0; /* set if using zonelist_cache */
  1419. int did_zlc_setup = 0; /* just call zlc_setup() one time */
  1420. classzone_idx = zone_idx(preferred_zone);
  1421. zonelist_scan:
  1422. /*
  1423. * Scan zonelist, looking for a zone with enough free.
  1424. * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
  1425. */
  1426. for_each_zone_zonelist_nodemask(zone, z, zonelist,
  1427. high_zoneidx, nodemask) {
  1428. if (NUMA_BUILD && zlc_active &&
  1429. !zlc_zone_worth_trying(zonelist, z, allowednodes))
  1430. continue;
  1431. if ((alloc_flags & ALLOC_CPUSET) &&
  1432. !cpuset_zone_allowed_softwall(zone, gfp_mask))
  1433. continue;
  1434. BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
  1435. if (!(alloc_flags & ALLOC_NO_WATERMARKS)) {
  1436. unsigned long mark;
  1437. int ret;
  1438. mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK];
  1439. if (zone_watermark_ok(zone, order, mark,
  1440. classzone_idx, alloc_flags))
  1441. goto try_this_zone;
  1442. if (NUMA_BUILD && !did_zlc_setup && nr_online_nodes > 1) {
  1443. /*
  1444. * we do zlc_setup if there are multiple nodes
  1445. * and before considering the first zone allowed
  1446. * by the cpuset.
  1447. */
  1448. allowednodes = zlc_setup(zonelist, alloc_flags);
  1449. zlc_active = 1;
  1450. did_zlc_setup = 1;
  1451. }
  1452. if (zone_reclaim_mode == 0)
  1453. goto this_zone_full;
  1454. /*
  1455. * As we may have just activated ZLC, check if the first
  1456. * eligible zone has failed zone_reclaim recently.
  1457. */
  1458. if (NUMA_BUILD && zlc_active &&
  1459. !zlc_zone_worth_trying(zonelist, z, allowednodes))
  1460. continue;
  1461. ret = zone_reclaim(zone, gfp_mask, order);
  1462. switch (ret) {
  1463. case ZONE_RECLAIM_NOSCAN:
  1464. /* did not scan */
  1465. continue;
  1466. case ZONE_RECLAIM_FULL:
  1467. /* scanned but unreclaimable */
  1468. continue;
  1469. default:
  1470. /* did we reclaim enough */
  1471. if (!zone_watermark_ok(zone, order, mark,
  1472. classzone_idx, alloc_flags))
  1473. goto this_zone_full;
  1474. }
  1475. }
  1476. try_this_zone:
  1477. page = buffered_rmqueue(preferred_zone, zone, order,
  1478. gfp_mask, migratetype);
  1479. if (page)
  1480. break;
  1481. this_zone_full:
  1482. if (NUMA_BUILD)
  1483. zlc_mark_zone_full(zonelist, z);
  1484. }
  1485. if (unlikely(NUMA_BUILD && page == NULL && zlc_active)) {
  1486. /* Disable zlc cache for second zonelist scan */
  1487. zlc_active = 0;
  1488. goto zonelist_scan;
  1489. }
  1490. return page;
  1491. }
  1492. /*
  1493. * Large machines with many possible nodes should not always dump per-node
  1494. * meminfo in irq context.
  1495. */
  1496. static inline bool should_suppress_show_mem(void)
  1497. {
  1498. bool ret = false;
  1499. #if NODES_SHIFT > 8
  1500. ret = in_interrupt();
  1501. #endif
  1502. return ret;
  1503. }
  1504. static DEFINE_RATELIMIT_STATE(nopage_rs,
  1505. DEFAULT_RATELIMIT_INTERVAL,
  1506. DEFAULT_RATELIMIT_BURST);
  1507. void warn_alloc_failed(gfp_t gfp_mask, int order, const char *fmt, ...)
  1508. {
  1509. va_list args;
  1510. unsigned int filter = SHOW_MEM_FILTER_NODES;
  1511. if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs))
  1512. return;
  1513. /*
  1514. * This documents exceptions given to allocations in certain
  1515. * contexts that are allowed to allocate outside current's set
  1516. * of allowed nodes.
  1517. */
  1518. if (!(gfp_mask & __GFP_NOMEMALLOC))
  1519. if (test_thread_flag(TIF_MEMDIE) ||
  1520. (current->flags & (PF_MEMALLOC | PF_EXITING)))
  1521. filter &= ~SHOW_MEM_FILTER_NODES;
  1522. if (in_interrupt() || !(gfp_mask & __GFP_WAIT))
  1523. filter &= ~SHOW_MEM_FILTER_NODES;
  1524. if (fmt) {
  1525. printk(KERN_WARNING);
  1526. va_start(args, fmt);
  1527. vprintk(fmt, args);
  1528. va_end(args);
  1529. }
  1530. pr_warning("%s: page allocation failure: order:%d, mode:0x%x\n",
  1531. current->comm, order, gfp_mask);
  1532. dump_stack();
  1533. if (!should_suppress_show_mem())
  1534. show_mem(filter);
  1535. }
  1536. static inline int
  1537. should_alloc_retry(gfp_t gfp_mask, unsigned int order,
  1538. unsigned long pages_reclaimed)
  1539. {
  1540. /* Do not loop if specifically requested */
  1541. if (gfp_mask & __GFP_NORETRY)
  1542. return 0;
  1543. /*
  1544. * In this implementation, order <= PAGE_ALLOC_COSTLY_ORDER
  1545. * means __GFP_NOFAIL, but that may not be true in other
  1546. * implementations.
  1547. */
  1548. if (order <= PAGE_ALLOC_COSTLY_ORDER)
  1549. return 1;
  1550. /*
  1551. * For order > PAGE_ALLOC_COSTLY_ORDER, if __GFP_REPEAT is
  1552. * specified, then we retry until we no longer reclaim any pages
  1553. * (above), or we've reclaimed an order of pages at least as
  1554. * large as the allocation's order. In both cases, if the
  1555. * allocation still fails, we stop retrying.
  1556. */
  1557. if (gfp_mask & __GFP_REPEAT && pages_reclaimed < (1 << order))
  1558. return 1;
  1559. /*
  1560. * Don't let big-order allocations loop unless the caller
  1561. * explicitly requests that.
  1562. */
  1563. if (gfp_mask & __GFP_NOFAIL)
  1564. return 1;
  1565. return 0;
  1566. }
  1567. static inline struct page *
  1568. __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
  1569. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1570. nodemask_t *nodemask, struct zone *preferred_zone,
  1571. int migratetype)
  1572. {
  1573. struct page *page;
  1574. /* Acquire the OOM killer lock for the zones in zonelist */
  1575. if (!try_set_zonelist_oom(zonelist, gfp_mask)) {
  1576. schedule_timeout_uninterruptible(1);
  1577. return NULL;
  1578. }
  1579. /*
  1580. * Go through the zonelist yet one more time, keep very high watermark
  1581. * here, this is only to catch a parallel oom killing, we must fail if
  1582. * we're still under heavy pressure.
  1583. */
  1584. page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask,
  1585. order, zonelist, high_zoneidx,
  1586. ALLOC_WMARK_HIGH|ALLOC_CPUSET,
  1587. preferred_zone, migratetype);
  1588. if (page)
  1589. goto out;
  1590. if (!(gfp_mask & __GFP_NOFAIL)) {
  1591. /* The OOM killer will not help higher order allocs */
  1592. if (order > PAGE_ALLOC_COSTLY_ORDER)
  1593. goto out;
  1594. /* The OOM killer does not needlessly kill tasks for lowmem */
  1595. if (high_zoneidx < ZONE_NORMAL)
  1596. goto out;
  1597. /*
  1598. * GFP_THISNODE contains __GFP_NORETRY and we never hit this.
  1599. * Sanity check for bare calls of __GFP_THISNODE, not real OOM.
  1600. * The caller should handle page allocation failure by itself if
  1601. * it specifies __GFP_THISNODE.
  1602. * Note: Hugepage uses it but will hit PAGE_ALLOC_COSTLY_ORDER.
  1603. */
  1604. if (gfp_mask & __GFP_THISNODE)
  1605. goto out;
  1606. }
  1607. /* Exhausted what can be done so it's blamo time */
  1608. out_of_memory(zonelist, gfp_mask, order, nodemask);
  1609. out:
  1610. clear_zonelist_oom(zonelist, gfp_mask);
  1611. return page;
  1612. }
  1613. #ifdef CONFIG_COMPACTION
  1614. /* Try memory compaction for high-order allocations before reclaim */
  1615. static struct page *
  1616. __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
  1617. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1618. nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
  1619. int migratetype, unsigned long *did_some_progress,
  1620. bool sync_migration)
  1621. {
  1622. struct page *page;
  1623. if (!order || compaction_deferred(preferred_zone))
  1624. return NULL;
  1625. current->flags |= PF_MEMALLOC;
  1626. *did_some_progress = try_to_compact_pages(zonelist, order, gfp_mask,
  1627. nodemask, sync_migration);
  1628. current->flags &= ~PF_MEMALLOC;
  1629. if (*did_some_progress != COMPACT_SKIPPED) {
  1630. /* Page migration frees to the PCP lists but we want merging */
  1631. drain_pages(get_cpu());
  1632. put_cpu();
  1633. page = get_page_from_freelist(gfp_mask, nodemask,
  1634. order, zonelist, high_zoneidx,
  1635. alloc_flags, preferred_zone,
  1636. migratetype);
  1637. if (page) {
  1638. preferred_zone->compact_considered = 0;
  1639. preferred_zone->compact_defer_shift = 0;
  1640. count_vm_event(COMPACTSUCCESS);
  1641. return page;
  1642. }
  1643. /*
  1644. * It's bad if compaction run occurs and fails.
  1645. * The most likely reason is that pages exist,
  1646. * but not enough to satisfy watermarks.
  1647. */
  1648. count_vm_event(COMPACTFAIL);
  1649. defer_compaction(preferred_zone);
  1650. cond_resched();
  1651. }
  1652. return NULL;
  1653. }
  1654. #else
  1655. static inline struct page *
  1656. __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
  1657. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1658. nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
  1659. int migratetype, unsigned long *did_some_progress,
  1660. bool sync_migration)
  1661. {
  1662. return NULL;
  1663. }
  1664. #endif /* CONFIG_COMPACTION */
  1665. /* The really slow allocator path where we enter direct reclaim */
  1666. static inline struct page *
  1667. __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
  1668. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1669. nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
  1670. int migratetype, unsigned long *did_some_progress)
  1671. {
  1672. struct page *page = NULL;
  1673. struct reclaim_state reclaim_state;
  1674. bool drained = false;
  1675. cond_resched();
  1676. /* We now go into synchronous reclaim */
  1677. cpuset_memory_pressure_bump();
  1678. current->flags |= PF_MEMALLOC;
  1679. lockdep_set_current_reclaim_state(gfp_mask);
  1680. reclaim_state.reclaimed_slab = 0;
  1681. current->reclaim_state = &reclaim_state;
  1682. *did_some_progress = try_to_free_pages(zonelist, order, gfp_mask, nodemask);
  1683. current->reclaim_state = NULL;
  1684. lockdep_clear_current_reclaim_state();
  1685. current->flags &= ~PF_MEMALLOC;
  1686. cond_resched();
  1687. if (unlikely(!(*did_some_progress)))
  1688. return NULL;
  1689. retry:
  1690. page = get_page_from_freelist(gfp_mask, nodemask, order,
  1691. zonelist, high_zoneidx,
  1692. alloc_flags, preferred_zone,
  1693. migratetype);
  1694. /*
  1695. * If an allocation failed after direct reclaim, it could be because
  1696. * pages are pinned on the per-cpu lists. Drain them and try again
  1697. */
  1698. if (!page && !drained) {
  1699. drain_all_pages();
  1700. drained = true;
  1701. goto retry;
  1702. }
  1703. return page;
  1704. }
  1705. /*
  1706. * This is called in the allocator slow-path if the allocation request is of
  1707. * sufficient urgency to ignore watermarks and take other desperate measures
  1708. */
  1709. static inline struct page *
  1710. __alloc_pages_high_priority(gfp_t gfp_mask, unsigned int order,
  1711. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1712. nodemask_t *nodemask, struct zone *preferred_zone,
  1713. int migratetype)
  1714. {
  1715. struct page *page;
  1716. do {
  1717. page = get_page_from_freelist(gfp_mask, nodemask, order,
  1718. zonelist, high_zoneidx, ALLOC_NO_WATERMARKS,
  1719. preferred_zone, migratetype);
  1720. if (!page && gfp_mask & __GFP_NOFAIL)
  1721. wait_iff_congested(preferred_zone, BLK_RW_ASYNC, HZ/50);
  1722. } while (!page && (gfp_mask & __GFP_NOFAIL));
  1723. return page;
  1724. }
  1725. static inline
  1726. void wake_all_kswapd(unsigned int order, struct zonelist *zonelist,
  1727. enum zone_type high_zoneidx,
  1728. enum zone_type classzone_idx)
  1729. {
  1730. struct zoneref *z;
  1731. struct zone *zone;
  1732. for_each_zone_zonelist(zone, z, zonelist, high_zoneidx)
  1733. wakeup_kswapd(zone, order, classzone_idx);
  1734. }
  1735. static inline int
  1736. gfp_to_alloc_flags(gfp_t gfp_mask)
  1737. {
  1738. int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
  1739. const gfp_t wait = gfp_mask & __GFP_WAIT;
  1740. /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */
  1741. BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);
  1742. /*
  1743. * The caller may dip into page reserves a bit more if the caller
  1744. * cannot run direct reclaim, or if the caller has realtime scheduling
  1745. * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will
  1746. * set both ALLOC_HARDER (!wait) and ALLOC_HIGH (__GFP_HIGH).
  1747. */
  1748. alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH);
  1749. if (!wait) {
  1750. /*
  1751. * Not worth trying to allocate harder for
  1752. * __GFP_NOMEMALLOC even if it can't schedule.
  1753. */
  1754. if (!(gfp_mask & __GFP_NOMEMALLOC))
  1755. alloc_flags |= ALLOC_HARDER;
  1756. /*
  1757. * Ignore cpuset if GFP_ATOMIC (!wait) rather than fail alloc.
  1758. * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
  1759. */
  1760. alloc_flags &= ~ALLOC_CPUSET;
  1761. } else if (unlikely(rt_task(current)) && !in_interrupt())
  1762. alloc_flags |= ALLOC_HARDER;
  1763. if (likely(!(gfp_mask & __GFP_NOMEMALLOC))) {
  1764. if (!in_interrupt() &&
  1765. ((current->flags & PF_MEMALLOC) ||
  1766. unlikely(test_thread_flag(TIF_MEMDIE))))
  1767. alloc_flags |= ALLOC_NO_WATERMARKS;
  1768. }
  1769. return alloc_flags;
  1770. }
  1771. static inline struct page *
  1772. __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
  1773. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1774. nodemask_t *nodemask, struct zone *preferred_zone,
  1775. int migratetype)
  1776. {
  1777. const gfp_t wait = gfp_mask & __GFP_WAIT;
  1778. struct page *page = NULL;
  1779. int alloc_flags;
  1780. unsigned long pages_reclaimed = 0;
  1781. unsigned long did_some_progress;
  1782. bool sync_migration = false;
  1783. /*
  1784. * In the slowpath, we sanity check order to avoid ever trying to
  1785. * reclaim >= MAX_ORDER areas which will never succeed. Callers may
  1786. * be using allocators in order of preference for an area that is
  1787. * too large.
  1788. */
  1789. if (order >= MAX_ORDER) {
  1790. WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));
  1791. return NULL;
  1792. }
  1793. /*
  1794. * GFP_THISNODE (meaning __GFP_THISNODE, __GFP_NORETRY and
  1795. * __GFP_NOWARN set) should not cause reclaim since the subsystem
  1796. * (f.e. slab) using GFP_THISNODE may choose to trigger reclaim
  1797. * using a larger set of nodes after it has established that the
  1798. * allowed per node queues are empty and that nodes are
  1799. * over allocated.
  1800. */
  1801. if (NUMA_BUILD && (gfp_mask & GFP_THISNODE) == GFP_THISNODE)
  1802. goto nopage;
  1803. restart:
  1804. if (!(gfp_mask & __GFP_NO_KSWAPD))
  1805. wake_all_kswapd(order, zonelist, high_zoneidx,
  1806. zone_idx(preferred_zone));
  1807. /*
  1808. * OK, we're below the kswapd watermark and have kicked background
  1809. * reclaim. Now things get more complex, so set up alloc_flags according
  1810. * to how we want to proceed.
  1811. */
  1812. alloc_flags = gfp_to_alloc_flags(gfp_mask);
  1813. /*
  1814. * Find the true preferred zone if the allocation is unconstrained by
  1815. * cpusets.
  1816. */
  1817. if (!(alloc_flags & ALLOC_CPUSET) && !nodemask)
  1818. first_zones_zonelist(zonelist, high_zoneidx, NULL,
  1819. &preferred_zone);
  1820. rebalance:
  1821. /* This is the last chance, in general, before the goto nopage. */
  1822. page = get_page_from_freelist(gfp_mask, nodemask, order, zonelist,
  1823. high_zoneidx, alloc_flags & ~ALLOC_NO_WATERMARKS,
  1824. preferred_zone, migratetype);
  1825. if (page)
  1826. goto got_pg;
  1827. /* Allocate without watermarks if the context allows */
  1828. if (alloc_flags & ALLOC_NO_WATERMARKS) {
  1829. page = __alloc_pages_high_priority(gfp_mask, order,
  1830. zonelist, high_zoneidx, nodemask,
  1831. preferred_zone, migratetype);
  1832. if (page)
  1833. goto got_pg;
  1834. }
  1835. /* Atomic allocations - we can't balance anything */
  1836. if (!wait)
  1837. goto nopage;
  1838. /* Avoid recursion of direct reclaim */
  1839. if (current->flags & PF_MEMALLOC)
  1840. goto nopage;
  1841. /* Avoid allocations with no watermarks from looping endlessly */
  1842. if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL))
  1843. goto nopage;
  1844. /*
  1845. * Try direct compaction. The first pass is asynchronous. Subsequent
  1846. * attempts after direct reclaim are synchronous
  1847. */
  1848. page = __alloc_pages_direct_compact(gfp_mask, order,
  1849. zonelist, high_zoneidx,
  1850. nodemask,
  1851. alloc_flags, preferred_zone,
  1852. migratetype, &did_some_progress,
  1853. sync_migration);
  1854. if (page)
  1855. goto got_pg;
  1856. sync_migration = true;
  1857. /* Try direct reclaim and then allocating */
  1858. page = __alloc_pages_direct_reclaim(gfp_mask, order,
  1859. zonelist, high_zoneidx,
  1860. nodemask,
  1861. alloc_flags, preferred_zone,
  1862. migratetype, &did_some_progress);
  1863. if (page)
  1864. goto got_pg;
  1865. /*
  1866. * If we failed to make any progress reclaiming, then we are
  1867. * running out of options and have to consider going OOM
  1868. */
  1869. if (!did_some_progress) {
  1870. if ((gfp_mask & __GFP_FS) && !(gfp_mask & __GFP_NORETRY)) {
  1871. if (oom_killer_disabled)
  1872. goto nopage;
  1873. page = __alloc_pages_may_oom(gfp_mask, order,
  1874. zonelist, high_zoneidx,
  1875. nodemask, preferred_zone,
  1876. migratetype);
  1877. if (page)
  1878. goto got_pg;
  1879. if (!(gfp_mask & __GFP_NOFAIL)) {
  1880. /*
  1881. * The oom killer is not called for high-order
  1882. * allocations that may fail, so if no progress
  1883. * is being made, there are no other options and
  1884. * retrying is unlikely to help.
  1885. */
  1886. if (order > PAGE_ALLOC_COSTLY_ORDER)
  1887. goto nopage;
  1888. /*
  1889. * The oom killer is not called for lowmem
  1890. * allocations to prevent needlessly killing
  1891. * innocent tasks.
  1892. */
  1893. if (high_zoneidx < ZONE_NORMAL)
  1894. goto nopage;
  1895. }
  1896. goto restart;
  1897. }
  1898. }
  1899. /* Check if we should retry the allocation */
  1900. pages_reclaimed += did_some_progress;
  1901. if (should_alloc_retry(gfp_mask, order, pages_reclaimed)) {
  1902. /* Wait for some write requests to complete then retry */
  1903. wait_iff_congested(preferred_zone, BLK_RW_ASYNC, HZ/50);
  1904. goto rebalance;
  1905. } else {
  1906. /*
  1907. * High-order allocations do not necessarily loop after
  1908. * direct reclaim and reclaim/compaction depends on compaction
  1909. * being called after reclaim so call directly if necessary
  1910. */
  1911. page = __alloc_pages_direct_compact(gfp_mask, order,
  1912. zonelist, high_zoneidx,
  1913. nodemask,
  1914. alloc_flags, preferred_zone,
  1915. migratetype, &did_some_progress,
  1916. sync_migration);
  1917. if (page)
  1918. goto got_pg;
  1919. }
  1920. nopage:
  1921. warn_alloc_failed(gfp_mask, order, NULL);
  1922. return page;
  1923. got_pg:
  1924. if (kmemcheck_enabled)
  1925. kmemcheck_pagealloc_alloc(page, order, gfp_mask);
  1926. return page;
  1927. }
  1928. /*
  1929. * This is the 'heart' of the zoned buddy allocator.
  1930. */
  1931. struct page *
  1932. __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
  1933. struct zonelist *zonelist, nodemask_t *nodemask)
  1934. {
  1935. enum zone_type high_zoneidx = gfp_zone(gfp_mask);
  1936. struct zone *preferred_zone;
  1937. struct page *page;
  1938. int migratetype = allocflags_to_migratetype(gfp_mask);
  1939. gfp_mask &= gfp_allowed_mask;
  1940. lockdep_trace_alloc(gfp_mask);
  1941. might_sleep_if(gfp_mask & __GFP_WAIT);
  1942. if (should_fail_alloc_page(gfp_mask, order))
  1943. return NULL;
  1944. /*
  1945. * Check the zones suitable for the gfp_mask contain at least one
  1946. * valid zone. It's possible to have an empty zonelist as a result
  1947. * of GFP_THISNODE and a memoryless node
  1948. */
  1949. if (unlikely(!zonelist->_zonerefs->zone))
  1950. return NULL;
  1951. get_mems_allowed();
  1952. /* The preferred zone is used for statistics later */
  1953. first_zones_zonelist(zonelist, high_zoneidx,
  1954. nodemask ? : &cpuset_current_mems_allowed,
  1955. &preferred_zone);
  1956. if (!preferred_zone) {
  1957. put_mems_allowed();
  1958. return NULL;
  1959. }
  1960. /* First allocation attempt */
  1961. page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask, order,
  1962. zonelist, high_zoneidx, ALLOC_WMARK_LOW|ALLOC_CPUSET,
  1963. preferred_zone, migratetype);
  1964. if (unlikely(!page))
  1965. page = __alloc_pages_slowpath(gfp_mask, order,
  1966. zonelist, high_zoneidx, nodemask,
  1967. preferred_zone, migratetype);
  1968. put_mems_allowed();
  1969. trace_mm_page_alloc(page, order, gfp_mask, migratetype);
  1970. return page;
  1971. }
  1972. EXPORT_SYMBOL(__alloc_pages_nodemask);
  1973. /*
  1974. * Common helper functions.
  1975. */
  1976. unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
  1977. {
  1978. struct page *page;
  1979. /*
  1980. * __get_free_pages() returns a 32-bit address, which cannot represent
  1981. * a highmem page
  1982. */
  1983. VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0);
  1984. page = alloc_pages(gfp_mask, order);
  1985. if (!page)
  1986. return 0;
  1987. return (unsigned long) page_address(page);
  1988. }
  1989. EXPORT_SYMBOL(__get_free_pages);
  1990. unsigned long get_zeroed_page(gfp_t gfp_mask)
  1991. {
  1992. return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
  1993. }
  1994. EXPORT_SYMBOL(get_zeroed_page);
  1995. void __pagevec_free(struct pagevec *pvec)
  1996. {
  1997. int i = pagevec_count(pvec);
  1998. while (--i >= 0) {
  1999. trace_mm_pagevec_free(pvec->pages[i], pvec->cold);
  2000. free_hot_cold_page(pvec->pages[i], pvec->cold);
  2001. }
  2002. }
  2003. void __free_pages(struct page *page, unsigned int order)
  2004. {
  2005. if (put_page_testzero(page)) {
  2006. if (order == 0)
  2007. free_hot_cold_page(page, 0);
  2008. else
  2009. __free_pages_ok(page, order);
  2010. }
  2011. }
  2012. EXPORT_SYMBOL(__free_pages);
  2013. void free_pages(unsigned long addr, unsigned int order)
  2014. {
  2015. if (addr != 0) {
  2016. VM_BUG_ON(!virt_addr_valid((void *)addr));
  2017. __free_pages(virt_to_page((void *)addr), order);
  2018. }
  2019. }
  2020. EXPORT_SYMBOL(free_pages);
  2021. static void *make_alloc_exact(unsigned long addr, unsigned order, size_t size)
  2022. {
  2023. if (addr) {
  2024. unsigned long alloc_end = addr + (PAGE_SIZE << order);
  2025. unsigned long used = addr + PAGE_ALIGN(size);
  2026. split_page(virt_to_page((void *)addr), order);
  2027. while (used < alloc_end) {
  2028. free_page(used);
  2029. used += PAGE_SIZE;
  2030. }
  2031. }
  2032. return (void *)addr;
  2033. }
  2034. /**
  2035. * alloc_pages_exact - allocate an exact number physically-contiguous pages.
  2036. * @size: the number of bytes to allocate
  2037. * @gfp_mask: GFP flags for the allocation
  2038. *
  2039. * This function is similar to alloc_pages(), except that it allocates the
  2040. * minimum number of pages to satisfy the request. alloc_pages() can only
  2041. * allocate memory in power-of-two pages.
  2042. *
  2043. * This function is also limited by MAX_ORDER.
  2044. *
  2045. * Memory allocated by this function must be released by free_pages_exact().
  2046. */
  2047. void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
  2048. {
  2049. unsigned int order = get_order(size);
  2050. unsigned long addr;
  2051. addr = __get_free_pages(gfp_mask, order);
  2052. return make_alloc_exact(addr, order, size);
  2053. }
  2054. EXPORT_SYMBOL(alloc_pages_exact);
  2055. /**
  2056. * alloc_pages_exact_nid - allocate an exact number of physically-contiguous
  2057. * pages on a node.
  2058. * @nid: the preferred node ID where memory should be allocated
  2059. * @size: the number of bytes to allocate
  2060. * @gfp_mask: GFP flags for the allocation
  2061. *
  2062. * Like alloc_pages_exact(), but try to allocate on node nid first before falling
  2063. * back.
  2064. * Note this is not alloc_pages_exact_node() which allocates on a specific node,
  2065. * but is not exact.
  2066. */
  2067. void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask)
  2068. {
  2069. unsigned order = get_order(size);
  2070. struct page *p = alloc_pages_node(nid, gfp_mask, order);
  2071. if (!p)
  2072. return NULL;
  2073. return make_alloc_exact((unsigned long)page_address(p), order, size);
  2074. }
  2075. EXPORT_SYMBOL(alloc_pages_exact_nid);
  2076. /**
  2077. * free_pages_exact - release memory allocated via alloc_pages_exact()
  2078. * @virt: the value returned by alloc_pages_exact.
  2079. * @size: size of allocation, same value as passed to alloc_pages_exact().
  2080. *
  2081. * Release the memory allocated by a previous call to alloc_pages_exact.
  2082. */
  2083. void free_pages_exact(void *virt, size_t size)
  2084. {
  2085. unsigned long addr = (unsigned long)virt;
  2086. unsigned long end = addr + PAGE_ALIGN(size);
  2087. while (addr < end) {
  2088. free_page(addr);
  2089. addr += PAGE_SIZE;
  2090. }
  2091. }
  2092. EXPORT_SYMBOL(free_pages_exact);
  2093. static unsigned int nr_free_zone_pages(int offset)
  2094. {
  2095. struct zoneref *z;
  2096. struct zone *zone;
  2097. /* Just pick one node, since fallback list is circular */
  2098. unsigned int sum = 0;
  2099. struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
  2100. for_each_zone_zonelist(zone, z, zonelist, offset) {
  2101. unsigned long size = zone->present_pages;
  2102. unsigned long high = high_wmark_pages(zone);
  2103. if (size > high)
  2104. sum += size - high;
  2105. }
  2106. return sum;
  2107. }
  2108. /*
  2109. * Amount of free RAM allocatable within ZONE_DMA and ZONE_NORMAL
  2110. */
  2111. unsigned int nr_free_buffer_pages(void)
  2112. {
  2113. return nr_free_zone_pages(gfp_zone(GFP_USER));
  2114. }
  2115. EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
  2116. /*
  2117. * Amount of free RAM allocatable within all zones
  2118. */
  2119. unsigned int nr_free_pagecache_pages(void)
  2120. {
  2121. return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
  2122. }
  2123. static inline void show_node(struct zone *zone)
  2124. {
  2125. if (NUMA_BUILD)
  2126. printk("Node %d ", zone_to_nid(zone));
  2127. }
  2128. void si_meminfo(struct sysinfo *val)
  2129. {
  2130. val->totalram = totalram_pages;
  2131. val->sharedram = 0;
  2132. val->freeram = global_page_state(NR_FREE_PAGES);
  2133. val->bufferram = nr_blockdev_pages();
  2134. val->totalhigh = totalhigh_pages;
  2135. val->freehigh = nr_free_highpages();
  2136. val->mem_unit = PAGE_SIZE;
  2137. }
  2138. EXPORT_SYMBOL(si_meminfo);
  2139. #ifdef CONFIG_NUMA
  2140. void si_meminfo_node(struct sysinfo *val, int nid)
  2141. {
  2142. pg_data_t *pgdat = NODE_DATA(nid);
  2143. val->totalram = pgdat->node_present_pages;
  2144. val->freeram = node_page_state(nid, NR_FREE_PAGES);
  2145. #ifdef CONFIG_HIGHMEM
  2146. val->totalhigh = pgdat->node_zones[ZONE_HIGHMEM].present_pages;
  2147. val->freehigh = zone_page_state(&pgdat->node_zones[ZONE_HIGHMEM],
  2148. NR_FREE_PAGES);
  2149. #else
  2150. val->totalhigh = 0;
  2151. val->freehigh = 0;
  2152. #endif
  2153. val->mem_unit = PAGE_SIZE;
  2154. }
  2155. #endif
  2156. /*
  2157. * Determine whether the node should be displayed or not, depending on whether
  2158. * SHOW_MEM_FILTER_NODES was passed to show_free_areas().
  2159. */
  2160. bool skip_free_areas_node(unsigned int flags, int nid)
  2161. {
  2162. bool ret = false;
  2163. if (!(flags & SHOW_MEM_FILTER_NODES))
  2164. goto out;
  2165. get_mems_allowed();
  2166. ret = !node_isset(nid, cpuset_current_mems_allowed);
  2167. put_mems_allowed();
  2168. out:
  2169. return ret;
  2170. }
  2171. #define K(x) ((x) << (PAGE_SHIFT-10))
  2172. /*
  2173. * Show free area list (used inside shift_scroll-lock stuff)
  2174. * We also calculate the percentage fragmentation. We do this by counting the
  2175. * memory on each free list with the exception of the first item on the list.
  2176. * Suppresses nodes that are not allowed by current's cpuset if
  2177. * SHOW_MEM_FILTER_NODES is passed.
  2178. */
  2179. void show_free_areas(unsigned int filter)
  2180. {
  2181. int cpu;
  2182. struct zone *zone;
  2183. for_each_populated_zone(zone) {
  2184. if (skip_free_areas_node(filter, zone_to_nid(zone)))
  2185. continue;
  2186. show_node(zone);
  2187. printk("%s per-cpu:\n", zone->name);
  2188. for_each_online_cpu(cpu) {
  2189. struct per_cpu_pageset *pageset;
  2190. pageset = per_cpu_ptr(zone->pageset, cpu);
  2191. printk("CPU %4d: hi:%5d, btch:%4d usd:%4d\n",
  2192. cpu, pageset->pcp.high,
  2193. pageset->pcp.batch, pageset->pcp.count);
  2194. }
  2195. }
  2196. printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n"
  2197. " active_file:%lu inactive_file:%lu isolated_file:%lu\n"
  2198. " unevictable:%lu"
  2199. " dirty:%lu writeback:%lu unstable:%lu\n"
  2200. " free:%lu slab_reclaimable:%lu slab_unreclaimable:%lu\n"
  2201. " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n",
  2202. global_page_state(NR_ACTIVE_ANON),
  2203. global_page_state(NR_INACTIVE_ANON),
  2204. global_page_state(NR_ISOLATED_ANON),
  2205. global_page_state(NR_ACTIVE_FILE),
  2206. global_page_state(NR_INACTIVE_FILE),
  2207. global_page_state(NR_ISOLATED_FILE),
  2208. global_page_state(NR_UNEVICTABLE),
  2209. global_page_state(NR_FILE_DIRTY),
  2210. global_page_state(NR_WRITEBACK),
  2211. global_page_state(NR_UNSTABLE_NFS),
  2212. global_page_state(NR_FREE_PAGES),
  2213. global_page_state(NR_SLAB_RECLAIMABLE),
  2214. global_page_state(NR_SLAB_UNRECLAIMABLE),
  2215. global_page_state(NR_FILE_MAPPED),
  2216. global_page_state(NR_SHMEM),
  2217. global_page_state(NR_PAGETABLE),
  2218. global_page_state(NR_BOUNCE));
  2219. for_each_populated_zone(zone) {
  2220. int i;
  2221. if (skip_free_areas_node(filter, zone_to_nid(zone)))
  2222. continue;
  2223. show_node(zone);
  2224. printk("%s"
  2225. " free:%lukB"
  2226. " min:%lukB"
  2227. " low:%lukB"
  2228. " high:%lukB"
  2229. " active_anon:%lukB"
  2230. " inactive_anon:%lukB"
  2231. " active_file:%lukB"
  2232. " inactive_file:%lukB"
  2233. " unevictable:%lukB"
  2234. " isolated(anon):%lukB"
  2235. " isolated(file):%lukB"
  2236. " present:%lukB"
  2237. " mlocked:%lukB"
  2238. " dirty:%lukB"
  2239. " writeback:%lukB"
  2240. " mapped:%lukB"
  2241. " shmem:%lukB"
  2242. " slab_reclaimable:%lukB"
  2243. " slab_unreclaimable:%lukB"
  2244. " kernel_stack:%lukB"
  2245. " pagetables:%lukB"
  2246. " unstable:%lukB"
  2247. " bounce:%lukB"
  2248. " writeback_tmp:%lukB"
  2249. " pages_scanned:%lu"
  2250. " all_unreclaimable? %s"
  2251. "\n",
  2252. zone->name,
  2253. K(zone_page_state(zone, NR_FREE_PAGES)),
  2254. K(min_wmark_pages(zone)),
  2255. K(low_wmark_pages(zone)),
  2256. K(high_wmark_pages(zone)),
  2257. K(zone_page_state(zone, NR_ACTIVE_ANON)),
  2258. K(zone_page_state(zone, NR_INACTIVE_ANON)),
  2259. K(zone_page_state(zone, NR_ACTIVE_FILE)),
  2260. K(zone_page_state(zone, NR_INACTIVE_FILE)),
  2261. K(zone_page_state(zone, NR_UNEVICTABLE)),
  2262. K(zone_page_state(zone, NR_ISOLATED_ANON)),
  2263. K(zone_page_state(zone, NR_ISOLATED_FILE)),
  2264. K(zone->present_pages),
  2265. K(zone_page_state(zone, NR_MLOCK)),
  2266. K(zone_page_state(zone, NR_FILE_DIRTY)),
  2267. K(zone_page_state(zone, NR_WRITEBACK)),
  2268. K(zone_page_state(zone, NR_FILE_MAPPED)),
  2269. K(zone_page_state(zone, NR_SHMEM)),
  2270. K(zone_page_state(zone, NR_SLAB_RECLAIMABLE)),
  2271. K(zone_page_state(zone, NR_SLAB_UNRECLAIMABLE)),
  2272. zone_page_state(zone, NR_KERNEL_STACK) *
  2273. THREAD_SIZE / 1024,
  2274. K(zone_page_state(zone, NR_PAGETABLE)),
  2275. K(zone_page_state(zone, NR_UNSTABLE_NFS)),
  2276. K(zone_page_state(zone, NR_BOUNCE)),
  2277. K(zone_page_state(zone, NR_WRITEBACK_TEMP)),
  2278. zone->pages_scanned,
  2279. (zone->all_unreclaimable ? "yes" : "no")
  2280. );
  2281. printk("lowmem_reserve[]:");
  2282. for (i = 0; i < MAX_NR_ZONES; i++)
  2283. printk(" %lu", zone->lowmem_reserve[i]);
  2284. printk("\n");
  2285. }
  2286. for_each_populated_zone(zone) {
  2287. unsigned long nr[MAX_ORDER], flags, order, total = 0;
  2288. if (skip_free_areas_node(filter, zone_to_nid(zone)))
  2289. continue;
  2290. show_node(zone);
  2291. printk("%s: ", zone->name);
  2292. spin_lock_irqsave(&zone->lock, flags);
  2293. for (order = 0; order < MAX_ORDER; order++) {
  2294. nr[order] = zone->free_area[order].nr_free;
  2295. total += nr[order] << order;
  2296. }
  2297. spin_unlock_irqrestore(&zone->lock, flags);
  2298. for (order = 0; order < MAX_ORDER; order++)
  2299. printk("%lu*%lukB ", nr[order], K(1UL) << order);
  2300. printk("= %lukB\n", K(total));
  2301. }
  2302. printk("%ld total pagecache pages\n", global_page_state(NR_FILE_PAGES));
  2303. show_swap_cache_info();
  2304. }
  2305. static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
  2306. {
  2307. zoneref->zone = zone;
  2308. zoneref->zone_idx = zone_idx(zone);
  2309. }
  2310. /*
  2311. * Builds allocation fallback zone lists.
  2312. *
  2313. * Add all populated zones of a node to the zonelist.
  2314. */
  2315. static int build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist,
  2316. int nr_zones, enum zone_type zone_type)
  2317. {
  2318. struct zone *zone;
  2319. BUG_ON(zone_type >= MAX_NR_ZONES);
  2320. zone_type++;
  2321. do {
  2322. zone_type--;
  2323. zone = pgdat->node_zones + zone_type;
  2324. if (populated_zone(zone)) {
  2325. zoneref_set_zone(zone,
  2326. &zonelist->_zonerefs[nr_zones++]);
  2327. check_highest_zone(zone_type);
  2328. }
  2329. } while (zone_type);
  2330. return nr_zones;
  2331. }
  2332. /*
  2333. * zonelist_order:
  2334. * 0 = automatic detection of better ordering.
  2335. * 1 = order by ([node] distance, -zonetype)
  2336. * 2 = order by (-zonetype, [node] distance)
  2337. *
  2338. * If not NUMA, ZONELIST_ORDER_ZONE and ZONELIST_ORDER_NODE will create
  2339. * the same zonelist. So only NUMA can configure this param.
  2340. */
  2341. #define ZONELIST_ORDER_DEFAULT 0
  2342. #define ZONELIST_ORDER_NODE 1
  2343. #define ZONELIST_ORDER_ZONE 2
  2344. /* zonelist order in the kernel.
  2345. * set_zonelist_order() will set this to NODE or ZONE.
  2346. */
  2347. static int current_zonelist_order = ZONELIST_ORDER_DEFAULT;
  2348. static char zonelist_order_name[3][8] = {"Default", "Node", "Zone"};
  2349. #ifdef CONFIG_NUMA
  2350. /* The value user specified ....changed by config */
  2351. static int user_zonelist_order = ZONELIST_ORDER_DEFAULT;
  2352. /* string for sysctl */
  2353. #define NUMA_ZONELIST_ORDER_LEN 16
  2354. char numa_zonelist_order[16] = "default";
  2355. /*
  2356. * interface for configure zonelist ordering.
  2357. * command line option "numa_zonelist_order"
  2358. * = "[dD]efault - default, automatic configuration.
  2359. * = "[nN]ode - order by node locality, then by zone within node
  2360. * = "[zZ]one - order by zone, then by locality within zone
  2361. */
  2362. static int __parse_numa_zonelist_order(char *s)
  2363. {
  2364. if (*s == 'd' || *s == 'D') {
  2365. user_zonelist_order = ZONELIST_ORDER_DEFAULT;
  2366. } else if (*s == 'n' || *s == 'N') {
  2367. user_zonelist_order = ZONELIST_ORDER_NODE;
  2368. } else if (*s == 'z' || *s == 'Z') {
  2369. user_zonelist_order = ZONELIST_ORDER_ZONE;
  2370. } else {
  2371. printk(KERN_WARNING
  2372. "Ignoring invalid numa_zonelist_order value: "
  2373. "%s\n", s);
  2374. return -EINVAL;
  2375. }
  2376. return 0;
  2377. }
  2378. static __init int setup_numa_zonelist_order(char *s)
  2379. {
  2380. int ret;
  2381. if (!s)
  2382. return 0;
  2383. ret = __parse_numa_zonelist_order(s);
  2384. if (ret == 0)
  2385. strlcpy(numa_zonelist_order, s, NUMA_ZONELIST_ORDER_LEN);
  2386. return ret;
  2387. }
  2388. early_param("numa_zonelist_order", setup_numa_zonelist_order);
  2389. /*
  2390. * sysctl handler for numa_zonelist_order
  2391. */
  2392. int numa_zonelist_order_handler(ctl_table *table, int write,
  2393. void __user *buffer, size_t *length,
  2394. loff_t *ppos)
  2395. {
  2396. char saved_string[NUMA_ZONELIST_ORDER_LEN];
  2397. int ret;
  2398. static DEFINE_MUTEX(zl_order_mutex);
  2399. mutex_lock(&zl_order_mutex);
  2400. if (write)
  2401. strcpy(saved_string, (char*)table->data);
  2402. ret = proc_dostring(table, write, buffer, length, ppos);
  2403. if (ret)
  2404. goto out;
  2405. if (write) {
  2406. int oldval = user_zonelist_order;
  2407. if (__parse_numa_zonelist_order((char*)table->data)) {
  2408. /*
  2409. * bogus value. restore saved string
  2410. */
  2411. strncpy((char*)table->data, saved_string,
  2412. NUMA_ZONELIST_ORDER_LEN);
  2413. user_zonelist_order = oldval;
  2414. } else if (oldval != user_zonelist_order) {
  2415. mutex_lock(&zonelists_mutex);
  2416. build_all_zonelists(NULL);
  2417. mutex_unlock(&zonelists_mutex);
  2418. }
  2419. }
  2420. out:
  2421. mutex_unlock(&zl_order_mutex);
  2422. return ret;
  2423. }
  2424. #define MAX_NODE_LOAD (nr_online_nodes)
  2425. static int node_load[MAX_NUMNODES];
  2426. /**
  2427. * find_next_best_node - find the next node that should appear in a given node's fallback list
  2428. * @node: node whose fallback list we're appending
  2429. * @used_node_mask: nodemask_t of already used nodes
  2430. *
  2431. * We use a number of factors to determine which is the next node that should
  2432. * appear on a given node's fallback list. The node should not have appeared
  2433. * already in @node's fallback list, and it should be the next closest node
  2434. * according to the distance array (which contains arbitrary distance values
  2435. * from each node to each node in the system), and should also prefer nodes
  2436. * with no CPUs, since presumably they'll have very little allocation pressure
  2437. * on them otherwise.
  2438. * It returns -1 if no node is found.
  2439. */
  2440. static int find_next_best_node(int node, nodemask_t *used_node_mask)
  2441. {
  2442. int n, val;
  2443. int min_val = INT_MAX;
  2444. int best_node = -1;
  2445. const struct cpumask *tmp = cpumask_of_node(0);
  2446. /* Use the local node if we haven't already */
  2447. if (!node_isset(node, *used_node_mask)) {
  2448. node_set(node, *used_node_mask);
  2449. return node;
  2450. }
  2451. for_each_node_state(n, N_HIGH_MEMORY) {
  2452. /* Don't want a node to appear more than once */
  2453. if (node_isset(n, *used_node_mask))
  2454. continue;
  2455. /* Use the distance array to find the distance */
  2456. val = node_distance(node, n);
  2457. /* Penalize nodes under us ("prefer the next node") */
  2458. val += (n < node);
  2459. /* Give preference to headless and unused nodes */
  2460. tmp = cpumask_of_node(n);
  2461. if (!cpumask_empty(tmp))
  2462. val += PENALTY_FOR_NODE_WITH_CPUS;
  2463. /* Slight preference for less loaded node */
  2464. val *= (MAX_NODE_LOAD*MAX_NUMNODES);
  2465. val += node_load[n];
  2466. if (val < min_val) {
  2467. min_val = val;
  2468. best_node = n;
  2469. }
  2470. }
  2471. if (best_node >= 0)
  2472. node_set(best_node, *used_node_mask);
  2473. return best_node;
  2474. }
  2475. /*
  2476. * Build zonelists ordered by node and zones within node.
  2477. * This results in maximum locality--normal zone overflows into local
  2478. * DMA zone, if any--but risks exhausting DMA zone.
  2479. */
  2480. static void build_zonelists_in_node_order(pg_data_t *pgdat, int node)
  2481. {
  2482. int j;
  2483. struct zonelist *zonelist;
  2484. zonelist = &pgdat->node_zonelists[0];
  2485. for (j = 0; zonelist->_zonerefs[j].zone != NULL; j++)
  2486. ;
  2487. j = build_zonelists_node(NODE_DATA(node), zonelist, j,
  2488. MAX_NR_ZONES - 1);
  2489. zonelist->_zonerefs[j].zone = NULL;
  2490. zonelist->_zonerefs[j].zone_idx = 0;
  2491. }
  2492. /*
  2493. * Build gfp_thisnode zonelists
  2494. */
  2495. static void build_thisnode_zonelists(pg_data_t *pgdat)
  2496. {
  2497. int j;
  2498. struct zonelist *zonelist;
  2499. zonelist = &pgdat->node_zonelists[1];
  2500. j = build_zonelists_node(pgdat, zonelist, 0, MAX_NR_ZONES - 1);
  2501. zonelist->_zonerefs[j].zone = NULL;
  2502. zonelist->_zonerefs[j].zone_idx = 0;
  2503. }
  2504. /*
  2505. * Build zonelists ordered by zone and nodes within zones.
  2506. * This results in conserving DMA zone[s] until all Normal memory is
  2507. * exhausted, but results in overflowing to remote node while memory
  2508. * may still exist in local DMA zone.
  2509. */
  2510. static int node_order[MAX_NUMNODES];
  2511. static void build_zonelists_in_zone_order(pg_data_t *pgdat, int nr_nodes)
  2512. {
  2513. int pos, j, node;
  2514. int zone_type; /* needs to be signed */
  2515. struct zone *z;
  2516. struct zonelist *zonelist;
  2517. zonelist = &pgdat->node_zonelists[0];
  2518. pos = 0;
  2519. for (zone_type = MAX_NR_ZONES - 1; zone_type >= 0; zone_type--) {
  2520. for (j = 0; j < nr_nodes; j++) {
  2521. node = node_order[j];
  2522. z = &NODE_DATA(node)->node_zones[zone_type];
  2523. if (populated_zone(z)) {
  2524. zoneref_set_zone(z,
  2525. &zonelist->_zonerefs[pos++]);
  2526. check_highest_zone(zone_type);
  2527. }
  2528. }
  2529. }
  2530. zonelist->_zonerefs[pos].zone = NULL;
  2531. zonelist->_zonerefs[pos].zone_idx = 0;
  2532. }
  2533. static int default_zonelist_order(void)
  2534. {
  2535. int nid, zone_type;
  2536. unsigned long low_kmem_size,total_size;
  2537. struct zone *z;
  2538. int average_size;
  2539. /*
  2540. * ZONE_DMA and ZONE_DMA32 can be very small area in the system.
  2541. * If they are really small and used heavily, the system can fall
  2542. * into OOM very easily.
  2543. * This function detect ZONE_DMA/DMA32 size and configures zone order.
  2544. */
  2545. /* Is there ZONE_NORMAL ? (ex. ppc has only DMA zone..) */
  2546. low_kmem_size = 0;
  2547. total_size = 0;
  2548. for_each_online_node(nid) {
  2549. for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
  2550. z = &NODE_DATA(nid)->node_zones[zone_type];
  2551. if (populated_zone(z)) {
  2552. if (zone_type < ZONE_NORMAL)
  2553. low_kmem_size += z->present_pages;
  2554. total_size += z->present_pages;
  2555. } else if (zone_type == ZONE_NORMAL) {
  2556. /*
  2557. * If any node has only lowmem, then node order
  2558. * is preferred to allow kernel allocations
  2559. * locally; otherwise, they can easily infringe
  2560. * on other nodes when there is an abundance of
  2561. * lowmem available to allocate from.
  2562. */
  2563. return ZONELIST_ORDER_NODE;
  2564. }
  2565. }
  2566. }
  2567. if (!low_kmem_size || /* there are no DMA area. */
  2568. low_kmem_size > total_size/2) /* DMA/DMA32 is big. */
  2569. return ZONELIST_ORDER_NODE;
  2570. /*
  2571. * look into each node's config.
  2572. * If there is a node whose DMA/DMA32 memory is very big area on
  2573. * local memory, NODE_ORDER may be suitable.
  2574. */
  2575. average_size = total_size /
  2576. (nodes_weight(node_states[N_HIGH_MEMORY]) + 1);
  2577. for_each_online_node(nid) {
  2578. low_kmem_size = 0;
  2579. total_size = 0;
  2580. for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
  2581. z = &NODE_DATA(nid)->node_zones[zone_type];
  2582. if (populated_zone(z)) {
  2583. if (zone_type < ZONE_NORMAL)
  2584. low_kmem_size += z->present_pages;
  2585. total_size += z->present_pages;
  2586. }
  2587. }
  2588. if (low_kmem_size &&
  2589. total_size > average_size && /* ignore small node */
  2590. low_kmem_size > total_size * 70/100)
  2591. return ZONELIST_ORDER_NODE;
  2592. }
  2593. return ZONELIST_ORDER_ZONE;
  2594. }
  2595. static void set_zonelist_order(void)
  2596. {
  2597. if (user_zonelist_order == ZONELIST_ORDER_DEFAULT)
  2598. current_zonelist_order = default_zonelist_order();
  2599. else
  2600. current_zonelist_order = user_zonelist_order;
  2601. }
  2602. static void build_zonelists(pg_data_t *pgdat)
  2603. {
  2604. int j, node, load;
  2605. enum zone_type i;
  2606. nodemask_t used_mask;
  2607. int local_node, prev_node;
  2608. struct zonelist *zonelist;
  2609. int order = current_zonelist_order;
  2610. /* initialize zonelists */
  2611. for (i = 0; i < MAX_ZONELISTS; i++) {
  2612. zonelist = pgdat->node_zonelists + i;
  2613. zonelist->_zonerefs[0].zone = NULL;
  2614. zonelist->_zonerefs[0].zone_idx = 0;
  2615. }
  2616. /* NUMA-aware ordering of nodes */
  2617. local_node = pgdat->node_id;
  2618. load = nr_online_nodes;
  2619. prev_node = local_node;
  2620. nodes_clear(used_mask);
  2621. memset(node_order, 0, sizeof(node_order));
  2622. j = 0;
  2623. while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
  2624. int distance = node_distance(local_node, node);
  2625. /*
  2626. * If another node is sufficiently far away then it is better
  2627. * to reclaim pages in a zone before going off node.
  2628. */
  2629. if (distance > RECLAIM_DISTANCE)
  2630. zone_reclaim_mode = 1;
  2631. /*
  2632. * We don't want to pressure a particular node.
  2633. * So adding penalty to the first node in same
  2634. * distance group to make it round-robin.
  2635. */
  2636. if (distance != node_distance(local_node, prev_node))
  2637. node_load[node] = load;
  2638. prev_node = node;
  2639. load--;
  2640. if (order == ZONELIST_ORDER_NODE)
  2641. build_zonelists_in_node_order(pgdat, node);
  2642. else
  2643. node_order[j++] = node; /* remember order */
  2644. }
  2645. if (order == ZONELIST_ORDER_ZONE) {
  2646. /* calculate node order -- i.e., DMA last! */
  2647. build_zonelists_in_zone_order(pgdat, j);
  2648. }
  2649. build_thisnode_zonelists(pgdat);
  2650. }
  2651. /* Construct the zonelist performance cache - see further mmzone.h */
  2652. static void build_zonelist_cache(pg_data_t *pgdat)
  2653. {
  2654. struct zonelist *zonelist;
  2655. struct zonelist_cache *zlc;
  2656. struct zoneref *z;
  2657. zonelist = &pgdat->node_zonelists[0];
  2658. zonelist->zlcache_ptr = zlc = &zonelist->zlcache;
  2659. bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
  2660. for (z = zonelist->_zonerefs; z->zone; z++)
  2661. zlc->z_to_n[z - zonelist->_zonerefs] = zonelist_node_idx(z);
  2662. }
  2663. #ifdef CONFIG_HAVE_MEMORYLESS_NODES
  2664. /*
  2665. * Return node id of node used for "local" allocations.
  2666. * I.e., first node id of first zone in arg node's generic zonelist.
  2667. * Used for initializing percpu 'numa_mem', which is used primarily
  2668. * for kernel allocations, so use GFP_KERNEL flags to locate zonelist.
  2669. */
  2670. int local_memory_node(int node)
  2671. {
  2672. struct zone *zone;
  2673. (void)first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
  2674. gfp_zone(GFP_KERNEL),
  2675. NULL,
  2676. &zone);
  2677. return zone->node;
  2678. }
  2679. #endif
  2680. #else /* CONFIG_NUMA */
  2681. static void set_zonelist_order(void)
  2682. {
  2683. current_zonelist_order = ZONELIST_ORDER_ZONE;
  2684. }
  2685. static void build_zonelists(pg_data_t *pgdat)
  2686. {
  2687. int node, local_node;
  2688. enum zone_type j;
  2689. struct zonelist *zonelist;
  2690. local_node = pgdat->node_id;
  2691. zonelist = &pgdat->node_zonelists[0];
  2692. j = build_zonelists_node(pgdat, zonelist, 0, MAX_NR_ZONES - 1);
  2693. /*
  2694. * Now we build the zonelist so that it contains the zones
  2695. * of all the other nodes.
  2696. * We don't want to pressure a particular node, so when
  2697. * building the zones for node N, we make sure that the
  2698. * zones coming right after the local ones are those from
  2699. * node N+1 (modulo N)
  2700. */
  2701. for (node = local_node + 1; node < MAX_NUMNODES; node++) {
  2702. if (!node_online(node))
  2703. continue;
  2704. j = build_zonelists_node(NODE_DATA(node), zonelist, j,
  2705. MAX_NR_ZONES - 1);
  2706. }
  2707. for (node = 0; node < local_node; node++) {
  2708. if (!node_online(node))
  2709. continue;
  2710. j = build_zonelists_node(NODE_DATA(node), zonelist, j,
  2711. MAX_NR_ZONES - 1);
  2712. }
  2713. zonelist->_zonerefs[j].zone = NULL;
  2714. zonelist->_zonerefs[j].zone_idx = 0;
  2715. }
  2716. /* non-NUMA variant of zonelist performance cache - just NULL zlcache_ptr */
  2717. static void build_zonelist_cache(pg_data_t *pgdat)
  2718. {
  2719. pgdat->node_zonelists[0].zlcache_ptr = NULL;
  2720. }
  2721. #endif /* CONFIG_NUMA */
  2722. /*
  2723. * Boot pageset table. One per cpu which is going to be used for all
  2724. * zones and all nodes. The parameters will be set in such a way
  2725. * that an item put on a list will immediately be handed over to
  2726. * the buddy list. This is safe since pageset manipulation is done
  2727. * with interrupts disabled.
  2728. *
  2729. * The boot_pagesets must be kept even after bootup is complete for
  2730. * unused processors and/or zones. They do play a role for bootstrapping
  2731. * hotplugged processors.
  2732. *
  2733. * zoneinfo_show() and maybe other functions do
  2734. * not check if the processor is online before following the pageset pointer.
  2735. * Other parts of the kernel may not check if the zone is available.
  2736. */
  2737. static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
  2738. static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
  2739. static void setup_zone_pageset(struct zone *zone);
  2740. /*
  2741. * Global mutex to protect against size modification of zonelists
  2742. * as well as to serialize pageset setup for the new populated zone.
  2743. */
  2744. DEFINE_MUTEX(zonelists_mutex);
  2745. /* return values int ....just for stop_machine() */
  2746. static __init_refok int __build_all_zonelists(void *data)
  2747. {
  2748. int nid;
  2749. int cpu;
  2750. #ifdef CONFIG_NUMA
  2751. memset(node_load, 0, sizeof(node_load));
  2752. #endif
  2753. for_each_online_node(nid) {
  2754. pg_data_t *pgdat = NODE_DATA(nid);
  2755. build_zonelists(pgdat);
  2756. build_zonelist_cache(pgdat);
  2757. }
  2758. /*
  2759. * Initialize the boot_pagesets that are going to be used
  2760. * for bootstrapping processors. The real pagesets for
  2761. * each zone will be allocated later when the per cpu
  2762. * allocator is available.
  2763. *
  2764. * boot_pagesets are used also for bootstrapping offline
  2765. * cpus if the system is already booted because the pagesets
  2766. * are needed to initialize allocators on a specific cpu too.
  2767. * F.e. the percpu allocator needs the page allocator which
  2768. * needs the percpu allocator in order to allocate its pagesets
  2769. * (a chicken-egg dilemma).
  2770. */
  2771. for_each_possible_cpu(cpu) {
  2772. setup_pageset(&per_cpu(boot_pageset, cpu), 0);
  2773. #ifdef CONFIG_HAVE_MEMORYLESS_NODES
  2774. /*
  2775. * We now know the "local memory node" for each node--
  2776. * i.e., the node of the first zone in the generic zonelist.
  2777. * Set up numa_mem percpu variable for on-line cpus. During
  2778. * boot, only the boot cpu should be on-line; we'll init the
  2779. * secondary cpus' numa_mem as they come on-line. During
  2780. * node/memory hotplug, we'll fixup all on-line cpus.
  2781. */
  2782. if (cpu_online(cpu))
  2783. set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu)));
  2784. #endif
  2785. }
  2786. return 0;
  2787. }
  2788. /*
  2789. * Called with zonelists_mutex held always
  2790. * unless system_state == SYSTEM_BOOTING.
  2791. */
  2792. void __ref build_all_zonelists(void *data)
  2793. {
  2794. set_zonelist_order();
  2795. if (system_state == SYSTEM_BOOTING) {
  2796. __build_all_zonelists(NULL);
  2797. mminit_verify_zonelist();
  2798. cpuset_init_current_mems_allowed();
  2799. } else {
  2800. /* we have to stop all cpus to guarantee there is no user
  2801. of zonelist */
  2802. #ifdef CONFIG_MEMORY_HOTPLUG
  2803. if (data)
  2804. setup_zone_pageset((struct zone *)data);
  2805. #endif
  2806. stop_machine(__build_all_zonelists, NULL, NULL);
  2807. /* cpuset refresh routine should be here */
  2808. }
  2809. vm_total_pages = nr_free_pagecache_pages();
  2810. /*
  2811. * Disable grouping by mobility if the number of pages in the
  2812. * system is too low to allow the mechanism to work. It would be
  2813. * more accurate, but expensive to check per-zone. This check is
  2814. * made on memory-hotadd so a system can start with mobility
  2815. * disabled and enable it later
  2816. */
  2817. if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
  2818. page_group_by_mobility_disabled = 1;
  2819. else
  2820. page_group_by_mobility_disabled = 0;
  2821. printk("Built %i zonelists in %s order, mobility grouping %s. "
  2822. "Total pages: %ld\n",
  2823. nr_online_nodes,
  2824. zonelist_order_name[current_zonelist_order],
  2825. page_group_by_mobility_disabled ? "off" : "on",
  2826. vm_total_pages);
  2827. #ifdef CONFIG_NUMA
  2828. printk("Policy zone: %s\n", zone_names[policy_zone]);
  2829. #endif
  2830. }
  2831. /*
  2832. * Helper functions to size the waitqueue hash table.
  2833. * Essentially these want to choose hash table sizes sufficiently
  2834. * large so that collisions trying to wait on pages are rare.
  2835. * But in fact, the number of active page waitqueues on typical
  2836. * systems is ridiculously low, less than 200. So this is even
  2837. * conservative, even though it seems large.
  2838. *
  2839. * The constant PAGES_PER_WAITQUEUE specifies the ratio of pages to
  2840. * waitqueues, i.e. the size of the waitq table given the number of pages.
  2841. */
  2842. #define PAGES_PER_WAITQUEUE 256
  2843. #ifndef CONFIG_MEMORY_HOTPLUG
  2844. static inline unsigned long wait_table_hash_nr_entries(unsigned long pages)
  2845. {
  2846. unsigned long size = 1;
  2847. pages /= PAGES_PER_WAITQUEUE;
  2848. while (size < pages)
  2849. size <<= 1;
  2850. /*
  2851. * Once we have dozens or even hundreds of threads sleeping
  2852. * on IO we've got bigger problems than wait queue collision.
  2853. * Limit the size of the wait table to a reasonable size.
  2854. */
  2855. size = min(size, 4096UL);
  2856. return max(size, 4UL);
  2857. }
  2858. #else
  2859. /*
  2860. * A zone's size might be changed by hot-add, so it is not possible to determine
  2861. * a suitable size for its wait_table. So we use the maximum size now.
  2862. *
  2863. * The max wait table size = 4096 x sizeof(wait_queue_head_t). ie:
  2864. *
  2865. * i386 (preemption config) : 4096 x 16 = 64Kbyte.
  2866. * ia64, x86-64 (no preemption): 4096 x 20 = 80Kbyte.
  2867. * ia64, x86-64 (preemption) : 4096 x 24 = 96Kbyte.
  2868. *
  2869. * The maximum entries are prepared when a zone's memory is (512K + 256) pages
  2870. * or more by the traditional way. (See above). It equals:
  2871. *
  2872. * i386, x86-64, powerpc(4K page size) : = ( 2G + 1M)byte.
  2873. * ia64(16K page size) : = ( 8G + 4M)byte.
  2874. * powerpc (64K page size) : = (32G +16M)byte.
  2875. */
  2876. static inline unsigned long wait_table_hash_nr_entries(unsigned long pages)
  2877. {
  2878. return 4096UL;
  2879. }
  2880. #endif
  2881. /*
  2882. * This is an integer logarithm so that shifts can be used later
  2883. * to extract the more random high bits from the multiplicative
  2884. * hash function before the remainder is taken.
  2885. */
  2886. static inline unsigned long wait_table_bits(unsigned long size)
  2887. {
  2888. return ffz(~size);
  2889. }
  2890. #define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1))
  2891. /*
  2892. * Check if a pageblock contains reserved pages
  2893. */
  2894. static int pageblock_is_reserved(unsigned long start_pfn, unsigned long end_pfn)
  2895. {
  2896. unsigned long pfn;
  2897. for (pfn = start_pfn; pfn < end_pfn; pfn++) {
  2898. if (!pfn_valid_within(pfn) || PageReserved(pfn_to_page(pfn)))
  2899. return 1;
  2900. }
  2901. return 0;
  2902. }
  2903. /*
  2904. * Mark a number of pageblocks as MIGRATE_RESERVE. The number
  2905. * of blocks reserved is based on min_wmark_pages(zone). The memory within
  2906. * the reserve will tend to store contiguous free pages. Setting min_free_kbytes
  2907. * higher will lead to a bigger reserve which will get freed as contiguous
  2908. * blocks as reclaim kicks in
  2909. */
  2910. static void setup_zone_migrate_reserve(struct zone *zone)
  2911. {
  2912. unsigned long start_pfn, pfn, end_pfn, block_end_pfn;
  2913. struct page *page;
  2914. unsigned long block_migratetype;
  2915. int reserve;
  2916. /* Get the start pfn, end pfn and the number of blocks to reserve */
  2917. start_pfn = zone->zone_start_pfn;
  2918. end_pfn = start_pfn + zone->spanned_pages;
  2919. reserve = roundup(min_wmark_pages(zone), pageblock_nr_pages) >>
  2920. pageblock_order;
  2921. /*
  2922. * Reserve blocks are generally in place to help high-order atomic
  2923. * allocations that are short-lived. A min_free_kbytes value that
  2924. * would result in more than 2 reserve blocks for atomic allocations
  2925. * is assumed to be in place to help anti-fragmentation for the
  2926. * future allocation of hugepages at runtime.
  2927. */
  2928. reserve = min(2, reserve);
  2929. for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
  2930. if (!pfn_valid(pfn))
  2931. continue;
  2932. page = pfn_to_page(pfn);
  2933. /* Watch out for overlapping nodes */
  2934. if (page_to_nid(page) != zone_to_nid(zone))
  2935. continue;
  2936. /* Blocks with reserved pages will never free, skip them. */
  2937. block_end_pfn = min(pfn + pageblock_nr_pages, end_pfn);
  2938. if (pageblock_is_reserved(pfn, block_end_pfn))
  2939. continue;
  2940. block_migratetype = get_pageblock_migratetype(page);
  2941. /* If this block is reserved, account for it */
  2942. if (reserve > 0 && block_migratetype == MIGRATE_RESERVE) {
  2943. reserve--;
  2944. continue;
  2945. }
  2946. /* Suitable for reserving if this block is movable */
  2947. if (reserve > 0 && block_migratetype == MIGRATE_MOVABLE) {
  2948. set_pageblock_migratetype(page, MIGRATE_RESERVE);
  2949. move_freepages_block(zone, page, MIGRATE_RESERVE);
  2950. reserve--;
  2951. continue;
  2952. }
  2953. /*
  2954. * If the reserve is met and this is a previous reserved block,
  2955. * take it back
  2956. */
  2957. if (block_migratetype == MIGRATE_RESERVE) {
  2958. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  2959. move_freepages_block(zone, page, MIGRATE_MOVABLE);
  2960. }
  2961. }
  2962. }
  2963. /*
  2964. * Initially all pages are reserved - free ones are freed
  2965. * up by free_all_bootmem() once the early boot process is
  2966. * done. Non-atomic initialization, single-pass.
  2967. */
  2968. void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
  2969. unsigned long start_pfn, enum memmap_context context)
  2970. {
  2971. struct page *page;
  2972. unsigned long end_pfn = start_pfn + size;
  2973. unsigned long pfn;
  2974. struct zone *z;
  2975. if (highest_memmap_pfn < end_pfn - 1)
  2976. highest_memmap_pfn = end_pfn - 1;
  2977. z = &NODE_DATA(nid)->node_zones[zone];
  2978. for (pfn = start_pfn; pfn < end_pfn; pfn++) {
  2979. /*
  2980. * There can be holes in boot-time mem_map[]s
  2981. * handed to this function. They do not
  2982. * exist on hotplugged memory.
  2983. */
  2984. if (context == MEMMAP_EARLY) {
  2985. if (!early_pfn_valid(pfn))
  2986. continue;
  2987. if (!early_pfn_in_nid(pfn, nid))
  2988. continue;
  2989. }
  2990. page = pfn_to_page(pfn);
  2991. set_page_links(page, zone, nid, pfn);
  2992. mminit_verify_page_links(page, zone, nid, pfn);
  2993. init_page_count(page);
  2994. reset_page_mapcount(page);
  2995. SetPageReserved(page);
  2996. /*
  2997. * Mark the block movable so that blocks are reserved for
  2998. * movable at startup. This will force kernel allocations
  2999. * to reserve their blocks rather than leaking throughout
  3000. * the address space during boot when many long-lived
  3001. * kernel allocations are made. Later some blocks near
  3002. * the start are marked MIGRATE_RESERVE by
  3003. * setup_zone_migrate_reserve()
  3004. *
  3005. * bitmap is created for zone's valid pfn range. but memmap
  3006. * can be created for invalid pages (for alignment)
  3007. * check here not to call set_pageblock_migratetype() against
  3008. * pfn out of zone.
  3009. */
  3010. if ((z->zone_start_pfn <= pfn)
  3011. && (pfn < z->zone_start_pfn + z->spanned_pages)
  3012. && !(pfn & (pageblock_nr_pages - 1)))
  3013. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  3014. INIT_LIST_HEAD(&page->lru);
  3015. #ifdef WANT_PAGE_VIRTUAL
  3016. /* The shift won't overflow because ZONE_NORMAL is below 4G. */
  3017. if (!is_highmem_idx(zone))
  3018. set_page_address(page, __va(pfn << PAGE_SHIFT));
  3019. #endif
  3020. }
  3021. }
  3022. static void __meminit zone_init_free_lists(struct zone *zone)
  3023. {
  3024. int order, t;
  3025. for_each_migratetype_order(order, t) {
  3026. INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
  3027. zone->free_area[order].nr_free = 0;
  3028. }
  3029. }
  3030. #ifndef __HAVE_ARCH_MEMMAP_INIT
  3031. #define memmap_init(size, nid, zone, start_pfn) \
  3032. memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY)
  3033. #endif
  3034. static int zone_batchsize(struct zone *zone)
  3035. {
  3036. #ifdef CONFIG_MMU
  3037. int batch;
  3038. /*
  3039. * The per-cpu-pages pools are set to around 1000th of the
  3040. * size of the zone. But no more than 1/2 of a meg.
  3041. *
  3042. * OK, so we don't know how big the cache is. So guess.
  3043. */
  3044. batch = zone->present_pages / 1024;
  3045. if (batch * PAGE_SIZE > 512 * 1024)
  3046. batch = (512 * 1024) / PAGE_SIZE;
  3047. batch /= 4; /* We effectively *= 4 below */
  3048. if (batch < 1)
  3049. batch = 1;
  3050. /*
  3051. * Clamp the batch to a 2^n - 1 value. Having a power
  3052. * of 2 value was found to be more likely to have
  3053. * suboptimal cache aliasing properties in some cases.
  3054. *
  3055. * For example if 2 tasks are alternately allocating
  3056. * batches of pages, one task can end up with a lot
  3057. * of pages of one half of the possible page colors
  3058. * and the other with pages of the other colors.
  3059. */
  3060. batch = rounddown_pow_of_two(batch + batch/2) - 1;
  3061. return batch;
  3062. #else
  3063. /* The deferral and batching of frees should be suppressed under NOMMU
  3064. * conditions.
  3065. *
  3066. * The problem is that NOMMU needs to be able to allocate large chunks
  3067. * of contiguous memory as there's no hardware page translation to
  3068. * assemble apparent contiguous memory from discontiguous pages.
  3069. *
  3070. * Queueing large contiguous runs of pages for batching, however,
  3071. * causes the pages to actually be freed in smaller chunks. As there
  3072. * can be a significant delay between the individual batches being
  3073. * recycled, this leads to the once large chunks of space being
  3074. * fragmented and becoming unavailable for high-order allocations.
  3075. */
  3076. return 0;
  3077. #endif
  3078. }
  3079. static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
  3080. {
  3081. struct per_cpu_pages *pcp;
  3082. int migratetype;
  3083. memset(p, 0, sizeof(*p));
  3084. pcp = &p->pcp;
  3085. pcp->count = 0;
  3086. pcp->high = 6 * batch;
  3087. pcp->batch = max(1UL, 1 * batch);
  3088. for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++)
  3089. INIT_LIST_HEAD(&pcp->lists[migratetype]);
  3090. }
  3091. /*
  3092. * setup_pagelist_highmark() sets the high water mark for hot per_cpu_pagelist
  3093. * to the value high for the pageset p.
  3094. */
  3095. static void setup_pagelist_highmark(struct per_cpu_pageset *p,
  3096. unsigned long high)
  3097. {
  3098. struct per_cpu_pages *pcp;
  3099. pcp = &p->pcp;
  3100. pcp->high = high;
  3101. pcp->batch = max(1UL, high/4);
  3102. if ((high/4) > (PAGE_SHIFT * 8))
  3103. pcp->batch = PAGE_SHIFT * 8;
  3104. }
  3105. static void setup_zone_pageset(struct zone *zone)
  3106. {
  3107. int cpu;
  3108. zone->pageset = alloc_percpu(struct per_cpu_pageset);
  3109. for_each_possible_cpu(cpu) {
  3110. struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);
  3111. setup_pageset(pcp, zone_batchsize(zone));
  3112. if (percpu_pagelist_fraction)
  3113. setup_pagelist_highmark(pcp,
  3114. (zone->present_pages /
  3115. percpu_pagelist_fraction));
  3116. }
  3117. }
  3118. /*
  3119. * Allocate per cpu pagesets and initialize them.
  3120. * Before this call only boot pagesets were available.
  3121. */
  3122. void __init setup_per_cpu_pageset(void)
  3123. {
  3124. struct zone *zone;
  3125. for_each_populated_zone(zone)
  3126. setup_zone_pageset(zone);
  3127. }
  3128. static noinline __init_refok
  3129. int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages)
  3130. {
  3131. int i;
  3132. struct pglist_data *pgdat = zone->zone_pgdat;
  3133. size_t alloc_size;
  3134. /*
  3135. * The per-page waitqueue mechanism uses hashed waitqueues
  3136. * per zone.
  3137. */
  3138. zone->wait_table_hash_nr_entries =
  3139. wait_table_hash_nr_entries(zone_size_pages);
  3140. zone->wait_table_bits =
  3141. wait_table_bits(zone->wait_table_hash_nr_entries);
  3142. alloc_size = zone->wait_table_hash_nr_entries
  3143. * sizeof(wait_queue_head_t);
  3144. if (!slab_is_available()) {
  3145. zone->wait_table = (wait_queue_head_t *)
  3146. alloc_bootmem_node_nopanic(pgdat, alloc_size);
  3147. } else {
  3148. /*
  3149. * This case means that a zone whose size was 0 gets new memory
  3150. * via memory hot-add.
  3151. * But it may be the case that a new node was hot-added. In
  3152. * this case vmalloc() will not be able to use this new node's
  3153. * memory - this wait_table must be initialized to use this new
  3154. * node itself as well.
  3155. * To use this new node's memory, further consideration will be
  3156. * necessary.
  3157. */
  3158. zone->wait_table = vmalloc(alloc_size);
  3159. }
  3160. if (!zone->wait_table)
  3161. return -ENOMEM;
  3162. for(i = 0; i < zone->wait_table_hash_nr_entries; ++i)
  3163. init_waitqueue_head(zone->wait_table + i);
  3164. return 0;
  3165. }
  3166. static int __zone_pcp_update(void *data)
  3167. {
  3168. struct zone *zone = data;
  3169. int cpu;
  3170. unsigned long batch = zone_batchsize(zone), flags;
  3171. for_each_possible_cpu(cpu) {
  3172. struct per_cpu_pageset *pset;
  3173. struct per_cpu_pages *pcp;
  3174. pset = per_cpu_ptr(zone->pageset, cpu);
  3175. pcp = &pset->pcp;
  3176. local_irq_save(flags);
  3177. free_pcppages_bulk(zone, pcp->count, pcp);
  3178. setup_pageset(pset, batch);
  3179. local_irq_restore(flags);
  3180. }
  3181. return 0;
  3182. }
  3183. void zone_pcp_update(struct zone *zone)
  3184. {
  3185. stop_machine(__zone_pcp_update, zone, NULL);
  3186. }
  3187. static __meminit void zone_pcp_init(struct zone *zone)
  3188. {
  3189. /*
  3190. * per cpu subsystem is not up at this point. The following code
  3191. * relies on the ability of the linker to provide the
  3192. * offset of a (static) per cpu variable into the per cpu area.
  3193. */
  3194. zone->pageset = &boot_pageset;
  3195. if (zone->present_pages)
  3196. printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n",
  3197. zone->name, zone->present_pages,
  3198. zone_batchsize(zone));
  3199. }
  3200. __meminit int init_currently_empty_zone(struct zone *zone,
  3201. unsigned long zone_start_pfn,
  3202. unsigned long size,
  3203. enum memmap_context context)
  3204. {
  3205. struct pglist_data *pgdat = zone->zone_pgdat;
  3206. int ret;
  3207. ret = zone_wait_table_init(zone, size);
  3208. if (ret)
  3209. return ret;
  3210. pgdat->nr_zones = zone_idx(zone) + 1;
  3211. zone->zone_start_pfn = zone_start_pfn;
  3212. mminit_dprintk(MMINIT_TRACE, "memmap_init",
  3213. "Initialising map node %d zone %lu pfns %lu -> %lu\n",
  3214. pgdat->node_id,
  3215. (unsigned long)zone_idx(zone),
  3216. zone_start_pfn, (zone_start_pfn + size));
  3217. zone_init_free_lists(zone);
  3218. return 0;
  3219. }
  3220. #ifdef CONFIG_ARCH_POPULATES_NODE_MAP
  3221. /*
  3222. * Basic iterator support. Return the first range of PFNs for a node
  3223. * Note: nid == MAX_NUMNODES returns first region regardless of node
  3224. */
  3225. static int __meminit first_active_region_index_in_nid(int nid)
  3226. {
  3227. int i;
  3228. for (i = 0; i < nr_nodemap_entries; i++)
  3229. if (nid == MAX_NUMNODES || early_node_map[i].nid == nid)
  3230. return i;
  3231. return -1;
  3232. }
  3233. /*
  3234. * Basic iterator support. Return the next active range of PFNs for a node
  3235. * Note: nid == MAX_NUMNODES returns next region regardless of node
  3236. */
  3237. static int __meminit next_active_region_index_in_nid(int index, int nid)
  3238. {
  3239. for (index = index + 1; index < nr_nodemap_entries; index++)
  3240. if (nid == MAX_NUMNODES || early_node_map[index].nid == nid)
  3241. return index;
  3242. return -1;
  3243. }
  3244. #ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
  3245. /*
  3246. * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
  3247. * Architectures may implement their own version but if add_active_range()
  3248. * was used and there are no special requirements, this is a convenient
  3249. * alternative
  3250. */
  3251. int __meminit __early_pfn_to_nid(unsigned long pfn)
  3252. {
  3253. int i;
  3254. for (i = 0; i < nr_nodemap_entries; i++) {
  3255. unsigned long start_pfn = early_node_map[i].start_pfn;
  3256. unsigned long end_pfn = early_node_map[i].end_pfn;
  3257. if (start_pfn <= pfn && pfn < end_pfn)
  3258. return early_node_map[i].nid;
  3259. }
  3260. /* This is a memory hole */
  3261. return -1;
  3262. }
  3263. #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
  3264. int __meminit early_pfn_to_nid(unsigned long pfn)
  3265. {
  3266. int nid;
  3267. nid = __early_pfn_to_nid(pfn);
  3268. if (nid >= 0)
  3269. return nid;
  3270. /* just returns 0 */
  3271. return 0;
  3272. }
  3273. #ifdef CONFIG_NODES_SPAN_OTHER_NODES
  3274. bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
  3275. {
  3276. int nid;
  3277. nid = __early_pfn_to_nid(pfn);
  3278. if (nid >= 0 && nid != node)
  3279. return false;
  3280. return true;
  3281. }
  3282. #endif
  3283. /* Basic iterator support to walk early_node_map[] */
  3284. #define for_each_active_range_index_in_nid(i, nid) \
  3285. for (i = first_active_region_index_in_nid(nid); i != -1; \
  3286. i = next_active_region_index_in_nid(i, nid))
  3287. /**
  3288. * free_bootmem_with_active_regions - Call free_bootmem_node for each active range
  3289. * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed.
  3290. * @max_low_pfn: The highest PFN that will be passed to free_bootmem_node
  3291. *
  3292. * If an architecture guarantees that all ranges registered with
  3293. * add_active_ranges() contain no holes and may be freed, this
  3294. * this function may be used instead of calling free_bootmem() manually.
  3295. */
  3296. void __init free_bootmem_with_active_regions(int nid,
  3297. unsigned long max_low_pfn)
  3298. {
  3299. int i;
  3300. for_each_active_range_index_in_nid(i, nid) {
  3301. unsigned long size_pages = 0;
  3302. unsigned long end_pfn = early_node_map[i].end_pfn;
  3303. if (early_node_map[i].start_pfn >= max_low_pfn)
  3304. continue;
  3305. if (end_pfn > max_low_pfn)
  3306. end_pfn = max_low_pfn;
  3307. size_pages = end_pfn - early_node_map[i].start_pfn;
  3308. free_bootmem_node(NODE_DATA(early_node_map[i].nid),
  3309. PFN_PHYS(early_node_map[i].start_pfn),
  3310. size_pages << PAGE_SHIFT);
  3311. }
  3312. }
  3313. #ifdef CONFIG_HAVE_MEMBLOCK
  3314. /*
  3315. * Basic iterator support. Return the last range of PFNs for a node
  3316. * Note: nid == MAX_NUMNODES returns last region regardless of node
  3317. */
  3318. static int __meminit last_active_region_index_in_nid(int nid)
  3319. {
  3320. int i;
  3321. for (i = nr_nodemap_entries - 1; i >= 0; i--)
  3322. if (nid == MAX_NUMNODES || early_node_map[i].nid == nid)
  3323. return i;
  3324. return -1;
  3325. }
  3326. /*
  3327. * Basic iterator support. Return the previous active range of PFNs for a node
  3328. * Note: nid == MAX_NUMNODES returns next region regardless of node
  3329. */
  3330. static int __meminit previous_active_region_index_in_nid(int index, int nid)
  3331. {
  3332. for (index = index - 1; index >= 0; index--)
  3333. if (nid == MAX_NUMNODES || early_node_map[index].nid == nid)
  3334. return index;
  3335. return -1;
  3336. }
  3337. #define for_each_active_range_index_in_nid_reverse(i, nid) \
  3338. for (i = last_active_region_index_in_nid(nid); i != -1; \
  3339. i = previous_active_region_index_in_nid(i, nid))
  3340. u64 __init find_memory_core_early(int nid, u64 size, u64 align,
  3341. u64 goal, u64 limit)
  3342. {
  3343. int i;
  3344. /* Need to go over early_node_map to find out good range for node */
  3345. for_each_active_range_index_in_nid_reverse(i, nid) {
  3346. u64 addr;
  3347. u64 ei_start, ei_last;
  3348. u64 final_start, final_end;
  3349. ei_last = early_node_map[i].end_pfn;
  3350. ei_last <<= PAGE_SHIFT;
  3351. ei_start = early_node_map[i].start_pfn;
  3352. ei_start <<= PAGE_SHIFT;
  3353. final_start = max(ei_start, goal);
  3354. final_end = min(ei_last, limit);
  3355. if (final_start >= final_end)
  3356. continue;
  3357. addr = memblock_find_in_range(final_start, final_end, size, align);
  3358. if (addr == MEMBLOCK_ERROR)
  3359. continue;
  3360. return addr;
  3361. }
  3362. return MEMBLOCK_ERROR;
  3363. }
  3364. #endif
  3365. int __init add_from_early_node_map(struct range *range, int az,
  3366. int nr_range, int nid)
  3367. {
  3368. int i;
  3369. u64 start, end;
  3370. /* need to go over early_node_map to find out good range for node */
  3371. for_each_active_range_index_in_nid(i, nid) {
  3372. start = early_node_map[i].start_pfn;
  3373. end = early_node_map[i].end_pfn;
  3374. nr_range = add_range(range, az, nr_range, start, end);
  3375. }
  3376. return nr_range;
  3377. }
  3378. void __init work_with_active_regions(int nid, work_fn_t work_fn, void *data)
  3379. {
  3380. int i;
  3381. int ret;
  3382. for_each_active_range_index_in_nid(i, nid) {
  3383. ret = work_fn(early_node_map[i].start_pfn,
  3384. early_node_map[i].end_pfn, data);
  3385. if (ret)
  3386. break;
  3387. }
  3388. }
  3389. /**
  3390. * sparse_memory_present_with_active_regions - Call memory_present for each active range
  3391. * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used.
  3392. *
  3393. * If an architecture guarantees that all ranges registered with
  3394. * add_active_ranges() contain no holes and may be freed, this
  3395. * function may be used instead of calling memory_present() manually.
  3396. */
  3397. void __init sparse_memory_present_with_active_regions(int nid)
  3398. {
  3399. int i;
  3400. for_each_active_range_index_in_nid(i, nid)
  3401. memory_present(early_node_map[i].nid,
  3402. early_node_map[i].start_pfn,
  3403. early_node_map[i].end_pfn);
  3404. }
  3405. /**
  3406. * get_pfn_range_for_nid - Return the start and end page frames for a node
  3407. * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
  3408. * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
  3409. * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
  3410. *
  3411. * It returns the start and end page frame of a node based on information
  3412. * provided by an arch calling add_active_range(). If called for a node
  3413. * with no available memory, a warning is printed and the start and end
  3414. * PFNs will be 0.
  3415. */
  3416. void __meminit get_pfn_range_for_nid(unsigned int nid,
  3417. unsigned long *start_pfn, unsigned long *end_pfn)
  3418. {
  3419. int i;
  3420. *start_pfn = -1UL;
  3421. *end_pfn = 0;
  3422. for_each_active_range_index_in_nid(i, nid) {
  3423. *start_pfn = min(*start_pfn, early_node_map[i].start_pfn);
  3424. *end_pfn = max(*end_pfn, early_node_map[i].end_pfn);
  3425. }
  3426. if (*start_pfn == -1UL)
  3427. *start_pfn = 0;
  3428. }
  3429. /*
  3430. * This finds a zone that can be used for ZONE_MOVABLE pages. The
  3431. * assumption is made that zones within a node are ordered in monotonic
  3432. * increasing memory addresses so that the "highest" populated zone is used
  3433. */
  3434. static void __init find_usable_zone_for_movable(void)
  3435. {
  3436. int zone_index;
  3437. for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
  3438. if (zone_index == ZONE_MOVABLE)
  3439. continue;
  3440. if (arch_zone_highest_possible_pfn[zone_index] >
  3441. arch_zone_lowest_possible_pfn[zone_index])
  3442. break;
  3443. }
  3444. VM_BUG_ON(zone_index == -1);
  3445. movable_zone = zone_index;
  3446. }
  3447. /*
  3448. * The zone ranges provided by the architecture do not include ZONE_MOVABLE
  3449. * because it is sized independent of architecture. Unlike the other zones,
  3450. * the starting point for ZONE_MOVABLE is not fixed. It may be different
  3451. * in each node depending on the size of each node and how evenly kernelcore
  3452. * is distributed. This helper function adjusts the zone ranges
  3453. * provided by the architecture for a given node by using the end of the
  3454. * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
  3455. * zones within a node are in order of monotonic increases memory addresses
  3456. */
  3457. static void __meminit adjust_zone_range_for_zone_movable(int nid,
  3458. unsigned long zone_type,
  3459. unsigned long node_start_pfn,
  3460. unsigned long node_end_pfn,
  3461. unsigned long *zone_start_pfn,
  3462. unsigned long *zone_end_pfn)
  3463. {
  3464. /* Only adjust if ZONE_MOVABLE is on this node */
  3465. if (zone_movable_pfn[nid]) {
  3466. /* Size ZONE_MOVABLE */
  3467. if (zone_type == ZONE_MOVABLE) {
  3468. *zone_start_pfn = zone_movable_pfn[nid];
  3469. *zone_end_pfn = min(node_end_pfn,
  3470. arch_zone_highest_possible_pfn[movable_zone]);
  3471. /* Adjust for ZONE_MOVABLE starting within this range */
  3472. } else if (*zone_start_pfn < zone_movable_pfn[nid] &&
  3473. *zone_end_pfn > zone_movable_pfn[nid]) {
  3474. *zone_end_pfn = zone_movable_pfn[nid];
  3475. /* Check if this whole range is within ZONE_MOVABLE */
  3476. } else if (*zone_start_pfn >= zone_movable_pfn[nid])
  3477. *zone_start_pfn = *zone_end_pfn;
  3478. }
  3479. }
  3480. /*
  3481. * Return the number of pages a zone spans in a node, including holes
  3482. * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
  3483. */
  3484. static unsigned long __meminit zone_spanned_pages_in_node(int nid,
  3485. unsigned long zone_type,
  3486. unsigned long *ignored)
  3487. {
  3488. unsigned long node_start_pfn, node_end_pfn;
  3489. unsigned long zone_start_pfn, zone_end_pfn;
  3490. /* Get the start and end of the node and zone */
  3491. get_pfn_range_for_nid(nid, &node_start_pfn, &node_end_pfn);
  3492. zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type];
  3493. zone_end_pfn = arch_zone_highest_possible_pfn[zone_type];
  3494. adjust_zone_range_for_zone_movable(nid, zone_type,
  3495. node_start_pfn, node_end_pfn,
  3496. &zone_start_pfn, &zone_end_pfn);
  3497. /* Check that this node has pages within the zone's required range */
  3498. if (zone_end_pfn < node_start_pfn || zone_start_pfn > node_end_pfn)
  3499. return 0;
  3500. /* Move the zone boundaries inside the node if necessary */
  3501. zone_end_pfn = min(zone_end_pfn, node_end_pfn);
  3502. zone_start_pfn = max(zone_start_pfn, node_start_pfn);
  3503. /* Return the spanned pages */
  3504. return zone_end_pfn - zone_start_pfn;
  3505. }
  3506. /*
  3507. * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
  3508. * then all holes in the requested range will be accounted for.
  3509. */
  3510. unsigned long __meminit __absent_pages_in_range(int nid,
  3511. unsigned long range_start_pfn,
  3512. unsigned long range_end_pfn)
  3513. {
  3514. int i = 0;
  3515. unsigned long prev_end_pfn = 0, hole_pages = 0;
  3516. unsigned long start_pfn;
  3517. /* Find the end_pfn of the first active range of pfns in the node */
  3518. i = first_active_region_index_in_nid(nid);
  3519. if (i == -1)
  3520. return 0;
  3521. prev_end_pfn = min(early_node_map[i].start_pfn, range_end_pfn);
  3522. /* Account for ranges before physical memory on this node */
  3523. if (early_node_map[i].start_pfn > range_start_pfn)
  3524. hole_pages = prev_end_pfn - range_start_pfn;
  3525. /* Find all holes for the zone within the node */
  3526. for (; i != -1; i = next_active_region_index_in_nid(i, nid)) {
  3527. /* No need to continue if prev_end_pfn is outside the zone */
  3528. if (prev_end_pfn >= range_end_pfn)
  3529. break;
  3530. /* Make sure the end of the zone is not within the hole */
  3531. start_pfn = min(early_node_map[i].start_pfn, range_end_pfn);
  3532. prev_end_pfn = max(prev_end_pfn, range_start_pfn);
  3533. /* Update the hole size cound and move on */
  3534. if (start_pfn > range_start_pfn) {
  3535. BUG_ON(prev_end_pfn > start_pfn);
  3536. hole_pages += start_pfn - prev_end_pfn;
  3537. }
  3538. prev_end_pfn = early_node_map[i].end_pfn;
  3539. }
  3540. /* Account for ranges past physical memory on this node */
  3541. if (range_end_pfn > prev_end_pfn)
  3542. hole_pages += range_end_pfn -
  3543. max(range_start_pfn, prev_end_pfn);
  3544. return hole_pages;
  3545. }
  3546. /**
  3547. * absent_pages_in_range - Return number of page frames in holes within a range
  3548. * @start_pfn: The start PFN to start searching for holes
  3549. * @end_pfn: The end PFN to stop searching for holes
  3550. *
  3551. * It returns the number of pages frames in memory holes within a range.
  3552. */
  3553. unsigned long __init absent_pages_in_range(unsigned long start_pfn,
  3554. unsigned long end_pfn)
  3555. {
  3556. return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
  3557. }
  3558. /* Return the number of page frames in holes in a zone on a node */
  3559. static unsigned long __meminit zone_absent_pages_in_node(int nid,
  3560. unsigned long zone_type,
  3561. unsigned long *ignored)
  3562. {
  3563. unsigned long node_start_pfn, node_end_pfn;
  3564. unsigned long zone_start_pfn, zone_end_pfn;
  3565. get_pfn_range_for_nid(nid, &node_start_pfn, &node_end_pfn);
  3566. zone_start_pfn = max(arch_zone_lowest_possible_pfn[zone_type],
  3567. node_start_pfn);
  3568. zone_end_pfn = min(arch_zone_highest_possible_pfn[zone_type],
  3569. node_end_pfn);
  3570. adjust_zone_range_for_zone_movable(nid, zone_type,
  3571. node_start_pfn, node_end_pfn,
  3572. &zone_start_pfn, &zone_end_pfn);
  3573. return __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
  3574. }
  3575. #else
  3576. static inline unsigned long __meminit zone_spanned_pages_in_node(int nid,
  3577. unsigned long zone_type,
  3578. unsigned long *zones_size)
  3579. {
  3580. return zones_size[zone_type];
  3581. }
  3582. static inline unsigned long __meminit zone_absent_pages_in_node(int nid,
  3583. unsigned long zone_type,
  3584. unsigned long *zholes_size)
  3585. {
  3586. if (!zholes_size)
  3587. return 0;
  3588. return zholes_size[zone_type];
  3589. }
  3590. #endif
  3591. static void __meminit calculate_node_totalpages(struct pglist_data *pgdat,
  3592. unsigned long *zones_size, unsigned long *zholes_size)
  3593. {
  3594. unsigned long realtotalpages, totalpages = 0;
  3595. enum zone_type i;
  3596. for (i = 0; i < MAX_NR_ZONES; i++)
  3597. totalpages += zone_spanned_pages_in_node(pgdat->node_id, i,
  3598. zones_size);
  3599. pgdat->node_spanned_pages = totalpages;
  3600. realtotalpages = totalpages;
  3601. for (i = 0; i < MAX_NR_ZONES; i++)
  3602. realtotalpages -=
  3603. zone_absent_pages_in_node(pgdat->node_id, i,
  3604. zholes_size);
  3605. pgdat->node_present_pages = realtotalpages;
  3606. printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id,
  3607. realtotalpages);
  3608. }
  3609. #ifndef CONFIG_SPARSEMEM
  3610. /*
  3611. * Calculate the size of the zone->blockflags rounded to an unsigned long
  3612. * Start by making sure zonesize is a multiple of pageblock_order by rounding
  3613. * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
  3614. * round what is now in bits to nearest long in bits, then return it in
  3615. * bytes.
  3616. */
  3617. static unsigned long __init usemap_size(unsigned long zonesize)
  3618. {
  3619. unsigned long usemapsize;
  3620. usemapsize = roundup(zonesize, pageblock_nr_pages);
  3621. usemapsize = usemapsize >> pageblock_order;
  3622. usemapsize *= NR_PAGEBLOCK_BITS;
  3623. usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
  3624. return usemapsize / 8;
  3625. }
  3626. static void __init setup_usemap(struct pglist_data *pgdat,
  3627. struct zone *zone, unsigned long zonesize)
  3628. {
  3629. unsigned long usemapsize = usemap_size(zonesize);
  3630. zone->pageblock_flags = NULL;
  3631. if (usemapsize)
  3632. zone->pageblock_flags = alloc_bootmem_node_nopanic(pgdat,
  3633. usemapsize);
  3634. }
  3635. #else
  3636. static inline void setup_usemap(struct pglist_data *pgdat,
  3637. struct zone *zone, unsigned long zonesize) {}
  3638. #endif /* CONFIG_SPARSEMEM */
  3639. #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
  3640. /* Return a sensible default order for the pageblock size. */
  3641. static inline int pageblock_default_order(void)
  3642. {
  3643. if (HPAGE_SHIFT > PAGE_SHIFT)
  3644. return HUGETLB_PAGE_ORDER;
  3645. return MAX_ORDER-1;
  3646. }
  3647. /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
  3648. static inline void __init set_pageblock_order(unsigned int order)
  3649. {
  3650. /* Check that pageblock_nr_pages has not already been setup */
  3651. if (pageblock_order)
  3652. return;
  3653. /*
  3654. * Assume the largest contiguous order of interest is a huge page.
  3655. * This value may be variable depending on boot parameters on IA64
  3656. */
  3657. pageblock_order = order;
  3658. }
  3659. #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
  3660. /*
  3661. * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
  3662. * and pageblock_default_order() are unused as pageblock_order is set
  3663. * at compile-time. See include/linux/pageblock-flags.h for the values of
  3664. * pageblock_order based on the kernel config
  3665. */
  3666. static inline int pageblock_default_order(unsigned int order)
  3667. {
  3668. return MAX_ORDER-1;
  3669. }
  3670. #define set_pageblock_order(x) do {} while (0)
  3671. #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
  3672. /*
  3673. * Set up the zone data structures:
  3674. * - mark all pages reserved
  3675. * - mark all memory queues empty
  3676. * - clear the memory bitmaps
  3677. */
  3678. static void __paginginit free_area_init_core(struct pglist_data *pgdat,
  3679. unsigned long *zones_size, unsigned long *zholes_size)
  3680. {
  3681. enum zone_type j;
  3682. int nid = pgdat->node_id;
  3683. unsigned long zone_start_pfn = pgdat->node_start_pfn;
  3684. int ret;
  3685. pgdat_resize_init(pgdat);
  3686. pgdat->nr_zones = 0;
  3687. init_waitqueue_head(&pgdat->kswapd_wait);
  3688. pgdat->kswapd_max_order = 0;
  3689. pgdat_page_cgroup_init(pgdat);
  3690. for (j = 0; j < MAX_NR_ZONES; j++) {
  3691. struct zone *zone = pgdat->node_zones + j;
  3692. unsigned long size, realsize, memmap_pages;
  3693. enum lru_list l;
  3694. size = zone_spanned_pages_in_node(nid, j, zones_size);
  3695. realsize = size - zone_absent_pages_in_node(nid, j,
  3696. zholes_size);
  3697. /*
  3698. * Adjust realsize so that it accounts for how much memory
  3699. * is used by this zone for memmap. This affects the watermark
  3700. * and per-cpu initialisations
  3701. */
  3702. memmap_pages =
  3703. PAGE_ALIGN(size * sizeof(struct page)) >> PAGE_SHIFT;
  3704. if (realsize >= memmap_pages) {
  3705. realsize -= memmap_pages;
  3706. if (memmap_pages)
  3707. printk(KERN_DEBUG
  3708. " %s zone: %lu pages used for memmap\n",
  3709. zone_names[j], memmap_pages);
  3710. } else
  3711. printk(KERN_WARNING
  3712. " %s zone: %lu pages exceeds realsize %lu\n",
  3713. zone_names[j], memmap_pages, realsize);
  3714. /* Account for reserved pages */
  3715. if (j == 0 && realsize > dma_reserve) {
  3716. realsize -= dma_reserve;
  3717. printk(KERN_DEBUG " %s zone: %lu pages reserved\n",
  3718. zone_names[0], dma_reserve);
  3719. }
  3720. if (!is_highmem_idx(j))
  3721. nr_kernel_pages += realsize;
  3722. nr_all_pages += realsize;
  3723. zone->spanned_pages = size;
  3724. zone->present_pages = realsize;
  3725. #ifdef CONFIG_NUMA
  3726. zone->node = nid;
  3727. zone->min_unmapped_pages = (realsize*sysctl_min_unmapped_ratio)
  3728. / 100;
  3729. zone->min_slab_pages = (realsize * sysctl_min_slab_ratio) / 100;
  3730. #endif
  3731. zone->name = zone_names[j];
  3732. spin_lock_init(&zone->lock);
  3733. spin_lock_init(&zone->lru_lock);
  3734. zone_seqlock_init(zone);
  3735. zone->zone_pgdat = pgdat;
  3736. zone_pcp_init(zone);
  3737. for_each_lru(l)
  3738. INIT_LIST_HEAD(&zone->lru[l].list);
  3739. zone->reclaim_stat.recent_rotated[0] = 0;
  3740. zone->reclaim_stat.recent_rotated[1] = 0;
  3741. zone->reclaim_stat.recent_scanned[0] = 0;
  3742. zone->reclaim_stat.recent_scanned[1] = 0;
  3743. zap_zone_vm_stats(zone);
  3744. zone->flags = 0;
  3745. if (!size)
  3746. continue;
  3747. set_pageblock_order(pageblock_default_order());
  3748. setup_usemap(pgdat, zone, size);
  3749. ret = init_currently_empty_zone(zone, zone_start_pfn,
  3750. size, MEMMAP_EARLY);
  3751. BUG_ON(ret);
  3752. memmap_init(size, nid, j, zone_start_pfn);
  3753. zone_start_pfn += size;
  3754. }
  3755. }
  3756. static void __init_refok alloc_node_mem_map(struct pglist_data *pgdat)
  3757. {
  3758. /* Skip empty nodes */
  3759. if (!pgdat->node_spanned_pages)
  3760. return;
  3761. #ifdef CONFIG_FLAT_NODE_MEM_MAP
  3762. /* ia64 gets its own node_mem_map, before this, without bootmem */
  3763. if (!pgdat->node_mem_map) {
  3764. unsigned long size, start, end;
  3765. struct page *map;
  3766. /*
  3767. * The zone's endpoints aren't required to be MAX_ORDER
  3768. * aligned but the node_mem_map endpoints must be in order
  3769. * for the buddy allocator to function correctly.
  3770. */
  3771. start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
  3772. end = pgdat->node_start_pfn + pgdat->node_spanned_pages;
  3773. end = ALIGN(end, MAX_ORDER_NR_PAGES);
  3774. size = (end - start) * sizeof(struct page);
  3775. map = alloc_remap(pgdat->node_id, size);
  3776. if (!map)
  3777. map = alloc_bootmem_node_nopanic(pgdat, size);
  3778. pgdat->node_mem_map = map + (pgdat->node_start_pfn - start);
  3779. }
  3780. #ifndef CONFIG_NEED_MULTIPLE_NODES
  3781. /*
  3782. * With no DISCONTIG, the global mem_map is just set as node 0's
  3783. */
  3784. if (pgdat == NODE_DATA(0)) {
  3785. mem_map = NODE_DATA(0)->node_mem_map;
  3786. #ifdef CONFIG_ARCH_POPULATES_NODE_MAP
  3787. if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
  3788. mem_map -= (pgdat->node_start_pfn - ARCH_PFN_OFFSET);
  3789. #endif /* CONFIG_ARCH_POPULATES_NODE_MAP */
  3790. }
  3791. #endif
  3792. #endif /* CONFIG_FLAT_NODE_MEM_MAP */
  3793. }
  3794. void __paginginit free_area_init_node(int nid, unsigned long *zones_size,
  3795. unsigned long node_start_pfn, unsigned long *zholes_size)
  3796. {
  3797. pg_data_t *pgdat = NODE_DATA(nid);
  3798. pgdat->node_id = nid;
  3799. pgdat->node_start_pfn = node_start_pfn;
  3800. calculate_node_totalpages(pgdat, zones_size, zholes_size);
  3801. alloc_node_mem_map(pgdat);
  3802. #ifdef CONFIG_FLAT_NODE_MEM_MAP
  3803. printk(KERN_DEBUG "free_area_init_node: node %d, pgdat %08lx, node_mem_map %08lx\n",
  3804. nid, (unsigned long)pgdat,
  3805. (unsigned long)pgdat->node_mem_map);
  3806. #endif
  3807. free_area_init_core(pgdat, zones_size, zholes_size);
  3808. }
  3809. #ifdef CONFIG_ARCH_POPULATES_NODE_MAP
  3810. #if MAX_NUMNODES > 1
  3811. /*
  3812. * Figure out the number of possible node ids.
  3813. */
  3814. static void __init setup_nr_node_ids(void)
  3815. {
  3816. unsigned int node;
  3817. unsigned int highest = 0;
  3818. for_each_node_mask(node, node_possible_map)
  3819. highest = node;
  3820. nr_node_ids = highest + 1;
  3821. }
  3822. #else
  3823. static inline void setup_nr_node_ids(void)
  3824. {
  3825. }
  3826. #endif
  3827. /**
  3828. * add_active_range - Register a range of PFNs backed by physical memory
  3829. * @nid: The node ID the range resides on
  3830. * @start_pfn: The start PFN of the available physical memory
  3831. * @end_pfn: The end PFN of the available physical memory
  3832. *
  3833. * These ranges are stored in an early_node_map[] and later used by
  3834. * free_area_init_nodes() to calculate zone sizes and holes. If the
  3835. * range spans a memory hole, it is up to the architecture to ensure
  3836. * the memory is not freed by the bootmem allocator. If possible
  3837. * the range being registered will be merged with existing ranges.
  3838. */
  3839. void __init add_active_range(unsigned int nid, unsigned long start_pfn,
  3840. unsigned long end_pfn)
  3841. {
  3842. int i;
  3843. mminit_dprintk(MMINIT_TRACE, "memory_register",
  3844. "Entering add_active_range(%d, %#lx, %#lx) "
  3845. "%d entries of %d used\n",
  3846. nid, start_pfn, end_pfn,
  3847. nr_nodemap_entries, MAX_ACTIVE_REGIONS);
  3848. mminit_validate_memmodel_limits(&start_pfn, &end_pfn);
  3849. /* Merge with existing active regions if possible */
  3850. for (i = 0; i < nr_nodemap_entries; i++) {
  3851. if (early_node_map[i].nid != nid)
  3852. continue;
  3853. /* Skip if an existing region covers this new one */
  3854. if (start_pfn >= early_node_map[i].start_pfn &&
  3855. end_pfn <= early_node_map[i].end_pfn)
  3856. return;
  3857. /* Merge forward if suitable */
  3858. if (start_pfn <= early_node_map[i].end_pfn &&
  3859. end_pfn > early_node_map[i].end_pfn) {
  3860. early_node_map[i].end_pfn = end_pfn;
  3861. return;
  3862. }
  3863. /* Merge backward if suitable */
  3864. if (start_pfn < early_node_map[i].start_pfn &&
  3865. end_pfn >= early_node_map[i].start_pfn) {
  3866. early_node_map[i].start_pfn = start_pfn;
  3867. return;
  3868. }
  3869. }
  3870. /* Check that early_node_map is large enough */
  3871. if (i >= MAX_ACTIVE_REGIONS) {
  3872. printk(KERN_CRIT "More than %d memory regions, truncating\n",
  3873. MAX_ACTIVE_REGIONS);
  3874. return;
  3875. }
  3876. early_node_map[i].nid = nid;
  3877. early_node_map[i].start_pfn = start_pfn;
  3878. early_node_map[i].end_pfn = end_pfn;
  3879. nr_nodemap_entries = i + 1;
  3880. }
  3881. /**
  3882. * remove_active_range - Shrink an existing registered range of PFNs
  3883. * @nid: The node id the range is on that should be shrunk
  3884. * @start_pfn: The new PFN of the range
  3885. * @end_pfn: The new PFN of the range
  3886. *
  3887. * i386 with NUMA use alloc_remap() to store a node_mem_map on a local node.
  3888. * The map is kept near the end physical page range that has already been
  3889. * registered. This function allows an arch to shrink an existing registered
  3890. * range.
  3891. */
  3892. void __init remove_active_range(unsigned int nid, unsigned long start_pfn,
  3893. unsigned long end_pfn)
  3894. {
  3895. int i, j;
  3896. int removed = 0;
  3897. printk(KERN_DEBUG "remove_active_range (%d, %lu, %lu)\n",
  3898. nid, start_pfn, end_pfn);
  3899. /* Find the old active region end and shrink */
  3900. for_each_active_range_index_in_nid(i, nid) {
  3901. if (early_node_map[i].start_pfn >= start_pfn &&
  3902. early_node_map[i].end_pfn <= end_pfn) {
  3903. /* clear it */
  3904. early_node_map[i].start_pfn = 0;
  3905. early_node_map[i].end_pfn = 0;
  3906. removed = 1;
  3907. continue;
  3908. }
  3909. if (early_node_map[i].start_pfn < start_pfn &&
  3910. early_node_map[i].end_pfn > start_pfn) {
  3911. unsigned long temp_end_pfn = early_node_map[i].end_pfn;
  3912. early_node_map[i].end_pfn = start_pfn;
  3913. if (temp_end_pfn > end_pfn)
  3914. add_active_range(nid, end_pfn, temp_end_pfn);
  3915. continue;
  3916. }
  3917. if (early_node_map[i].start_pfn >= start_pfn &&
  3918. early_node_map[i].end_pfn > end_pfn &&
  3919. early_node_map[i].start_pfn < end_pfn) {
  3920. early_node_map[i].start_pfn = end_pfn;
  3921. continue;
  3922. }
  3923. }
  3924. if (!removed)
  3925. return;
  3926. /* remove the blank ones */
  3927. for (i = nr_nodemap_entries - 1; i > 0; i--) {
  3928. if (early_node_map[i].nid != nid)
  3929. continue;
  3930. if (early_node_map[i].end_pfn)
  3931. continue;
  3932. /* we found it, get rid of it */
  3933. for (j = i; j < nr_nodemap_entries - 1; j++)
  3934. memcpy(&early_node_map[j], &early_node_map[j+1],
  3935. sizeof(early_node_map[j]));
  3936. j = nr_nodemap_entries - 1;
  3937. memset(&early_node_map[j], 0, sizeof(early_node_map[j]));
  3938. nr_nodemap_entries--;
  3939. }
  3940. }
  3941. /**
  3942. * remove_all_active_ranges - Remove all currently registered regions
  3943. *
  3944. * During discovery, it may be found that a table like SRAT is invalid
  3945. * and an alternative discovery method must be used. This function removes
  3946. * all currently registered regions.
  3947. */
  3948. void __init remove_all_active_ranges(void)
  3949. {
  3950. memset(early_node_map, 0, sizeof(early_node_map));
  3951. nr_nodemap_entries = 0;
  3952. }
  3953. /* Compare two active node_active_regions */
  3954. static int __init cmp_node_active_region(const void *a, const void *b)
  3955. {
  3956. struct node_active_region *arange = (struct node_active_region *)a;
  3957. struct node_active_region *brange = (struct node_active_region *)b;
  3958. /* Done this way to avoid overflows */
  3959. if (arange->start_pfn > brange->start_pfn)
  3960. return 1;
  3961. if (arange->start_pfn < brange->start_pfn)
  3962. return -1;
  3963. return 0;
  3964. }
  3965. /* sort the node_map by start_pfn */
  3966. void __init sort_node_map(void)
  3967. {
  3968. sort(early_node_map, (size_t)nr_nodemap_entries,
  3969. sizeof(struct node_active_region),
  3970. cmp_node_active_region, NULL);
  3971. }
  3972. /**
  3973. * node_map_pfn_alignment - determine the maximum internode alignment
  3974. *
  3975. * This function should be called after node map is populated and sorted.
  3976. * It calculates the maximum power of two alignment which can distinguish
  3977. * all the nodes.
  3978. *
  3979. * For example, if all nodes are 1GiB and aligned to 1GiB, the return value
  3980. * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the
  3981. * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is
  3982. * shifted, 1GiB is enough and this function will indicate so.
  3983. *
  3984. * This is used to test whether pfn -> nid mapping of the chosen memory
  3985. * model has fine enough granularity to avoid incorrect mapping for the
  3986. * populated node map.
  3987. *
  3988. * Returns the determined alignment in pfn's. 0 if there is no alignment
  3989. * requirement (single node).
  3990. */
  3991. unsigned long __init node_map_pfn_alignment(void)
  3992. {
  3993. unsigned long accl_mask = 0, last_end = 0;
  3994. int last_nid = -1;
  3995. int i;
  3996. for_each_active_range_index_in_nid(i, MAX_NUMNODES) {
  3997. int nid = early_node_map[i].nid;
  3998. unsigned long start = early_node_map[i].start_pfn;
  3999. unsigned long end = early_node_map[i].end_pfn;
  4000. unsigned long mask;
  4001. if (!start || last_nid < 0 || last_nid == nid) {
  4002. last_nid = nid;
  4003. last_end = end;
  4004. continue;
  4005. }
  4006. /*
  4007. * Start with a mask granular enough to pin-point to the
  4008. * start pfn and tick off bits one-by-one until it becomes
  4009. * too coarse to separate the current node from the last.
  4010. */
  4011. mask = ~((1 << __ffs(start)) - 1);
  4012. while (mask && last_end <= (start & (mask << 1)))
  4013. mask <<= 1;
  4014. /* accumulate all internode masks */
  4015. accl_mask |= mask;
  4016. }
  4017. /* convert mask to number of pages */
  4018. return ~accl_mask + 1;
  4019. }
  4020. /* Find the lowest pfn for a node */
  4021. static unsigned long __init find_min_pfn_for_node(int nid)
  4022. {
  4023. int i;
  4024. unsigned long min_pfn = ULONG_MAX;
  4025. /* Assuming a sorted map, the first range found has the starting pfn */
  4026. for_each_active_range_index_in_nid(i, nid)
  4027. min_pfn = min(min_pfn, early_node_map[i].start_pfn);
  4028. if (min_pfn == ULONG_MAX) {
  4029. printk(KERN_WARNING
  4030. "Could not find start_pfn for node %d\n", nid);
  4031. return 0;
  4032. }
  4033. return min_pfn;
  4034. }
  4035. /**
  4036. * find_min_pfn_with_active_regions - Find the minimum PFN registered
  4037. *
  4038. * It returns the minimum PFN based on information provided via
  4039. * add_active_range().
  4040. */
  4041. unsigned long __init find_min_pfn_with_active_regions(void)
  4042. {
  4043. return find_min_pfn_for_node(MAX_NUMNODES);
  4044. }
  4045. /*
  4046. * early_calculate_totalpages()
  4047. * Sum pages in active regions for movable zone.
  4048. * Populate N_HIGH_MEMORY for calculating usable_nodes.
  4049. */
  4050. static unsigned long __init early_calculate_totalpages(void)
  4051. {
  4052. int i;
  4053. unsigned long totalpages = 0;
  4054. for (i = 0; i < nr_nodemap_entries; i++) {
  4055. unsigned long pages = early_node_map[i].end_pfn -
  4056. early_node_map[i].start_pfn;
  4057. totalpages += pages;
  4058. if (pages)
  4059. node_set_state(early_node_map[i].nid, N_HIGH_MEMORY);
  4060. }
  4061. return totalpages;
  4062. }
  4063. /*
  4064. * Find the PFN the Movable zone begins in each node. Kernel memory
  4065. * is spread evenly between nodes as long as the nodes have enough
  4066. * memory. When they don't, some nodes will have more kernelcore than
  4067. * others
  4068. */
  4069. static void __init find_zone_movable_pfns_for_nodes(unsigned long *movable_pfn)
  4070. {
  4071. int i, nid;
  4072. unsigned long usable_startpfn;
  4073. unsigned long kernelcore_node, kernelcore_remaining;
  4074. /* save the state before borrow the nodemask */
  4075. nodemask_t saved_node_state = node_states[N_HIGH_MEMORY];
  4076. unsigned long totalpages = early_calculate_totalpages();
  4077. int usable_nodes = nodes_weight(node_states[N_HIGH_MEMORY]);
  4078. /*
  4079. * If movablecore was specified, calculate what size of
  4080. * kernelcore that corresponds so that memory usable for
  4081. * any allocation type is evenly spread. If both kernelcore
  4082. * and movablecore are specified, then the value of kernelcore
  4083. * will be used for required_kernelcore if it's greater than
  4084. * what movablecore would have allowed.
  4085. */
  4086. if (required_movablecore) {
  4087. unsigned long corepages;
  4088. /*
  4089. * Round-up so that ZONE_MOVABLE is at least as large as what
  4090. * was requested by the user
  4091. */
  4092. required_movablecore =
  4093. roundup(required_movablecore, MAX_ORDER_NR_PAGES);
  4094. corepages = totalpages - required_movablecore;
  4095. required_kernelcore = max(required_kernelcore, corepages);
  4096. }
  4097. /* If kernelcore was not specified, there is no ZONE_MOVABLE */
  4098. if (!required_kernelcore)
  4099. goto out;
  4100. /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
  4101. find_usable_zone_for_movable();
  4102. usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
  4103. restart:
  4104. /* Spread kernelcore memory as evenly as possible throughout nodes */
  4105. kernelcore_node = required_kernelcore / usable_nodes;
  4106. for_each_node_state(nid, N_HIGH_MEMORY) {
  4107. /*
  4108. * Recalculate kernelcore_node if the division per node
  4109. * now exceeds what is necessary to satisfy the requested
  4110. * amount of memory for the kernel
  4111. */
  4112. if (required_kernelcore < kernelcore_node)
  4113. kernelcore_node = required_kernelcore / usable_nodes;
  4114. /*
  4115. * As the map is walked, we track how much memory is usable
  4116. * by the kernel using kernelcore_remaining. When it is
  4117. * 0, the rest of the node is usable by ZONE_MOVABLE
  4118. */
  4119. kernelcore_remaining = kernelcore_node;
  4120. /* Go through each range of PFNs within this node */
  4121. for_each_active_range_index_in_nid(i, nid) {
  4122. unsigned long start_pfn, end_pfn;
  4123. unsigned long size_pages;
  4124. start_pfn = max(early_node_map[i].start_pfn,
  4125. zone_movable_pfn[nid]);
  4126. end_pfn = early_node_map[i].end_pfn;
  4127. if (start_pfn >= end_pfn)
  4128. continue;
  4129. /* Account for what is only usable for kernelcore */
  4130. if (start_pfn < usable_startpfn) {
  4131. unsigned long kernel_pages;
  4132. kernel_pages = min(end_pfn, usable_startpfn)
  4133. - start_pfn;
  4134. kernelcore_remaining -= min(kernel_pages,
  4135. kernelcore_remaining);
  4136. required_kernelcore -= min(kernel_pages,
  4137. required_kernelcore);
  4138. /* Continue if range is now fully accounted */
  4139. if (end_pfn <= usable_startpfn) {
  4140. /*
  4141. * Push zone_movable_pfn to the end so
  4142. * that if we have to rebalance
  4143. * kernelcore across nodes, we will
  4144. * not double account here
  4145. */
  4146. zone_movable_pfn[nid] = end_pfn;
  4147. continue;
  4148. }
  4149. start_pfn = usable_startpfn;
  4150. }
  4151. /*
  4152. * The usable PFN range for ZONE_MOVABLE is from
  4153. * start_pfn->end_pfn. Calculate size_pages as the
  4154. * number of pages used as kernelcore
  4155. */
  4156. size_pages = end_pfn - start_pfn;
  4157. if (size_pages > kernelcore_remaining)
  4158. size_pages = kernelcore_remaining;
  4159. zone_movable_pfn[nid] = start_pfn + size_pages;
  4160. /*
  4161. * Some kernelcore has been met, update counts and
  4162. * break if the kernelcore for this node has been
  4163. * satisified
  4164. */
  4165. required_kernelcore -= min(required_kernelcore,
  4166. size_pages);
  4167. kernelcore_remaining -= size_pages;
  4168. if (!kernelcore_remaining)
  4169. break;
  4170. }
  4171. }
  4172. /*
  4173. * If there is still required_kernelcore, we do another pass with one
  4174. * less node in the count. This will push zone_movable_pfn[nid] further
  4175. * along on the nodes that still have memory until kernelcore is
  4176. * satisified
  4177. */
  4178. usable_nodes--;
  4179. if (usable_nodes && required_kernelcore > usable_nodes)
  4180. goto restart;
  4181. /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
  4182. for (nid = 0; nid < MAX_NUMNODES; nid++)
  4183. zone_movable_pfn[nid] =
  4184. roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
  4185. out:
  4186. /* restore the node_state */
  4187. node_states[N_HIGH_MEMORY] = saved_node_state;
  4188. }
  4189. /* Any regular memory on that node ? */
  4190. static void check_for_regular_memory(pg_data_t *pgdat)
  4191. {
  4192. #ifdef CONFIG_HIGHMEM
  4193. enum zone_type zone_type;
  4194. for (zone_type = 0; zone_type <= ZONE_NORMAL; zone_type++) {
  4195. struct zone *zone = &pgdat->node_zones[zone_type];
  4196. if (zone->present_pages)
  4197. node_set_state(zone_to_nid(zone), N_NORMAL_MEMORY);
  4198. }
  4199. #endif
  4200. }
  4201. /**
  4202. * free_area_init_nodes - Initialise all pg_data_t and zone data
  4203. * @max_zone_pfn: an array of max PFNs for each zone
  4204. *
  4205. * This will call free_area_init_node() for each active node in the system.
  4206. * Using the page ranges provided by add_active_range(), the size of each
  4207. * zone in each node and their holes is calculated. If the maximum PFN
  4208. * between two adjacent zones match, it is assumed that the zone is empty.
  4209. * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
  4210. * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
  4211. * starts where the previous one ended. For example, ZONE_DMA32 starts
  4212. * at arch_max_dma_pfn.
  4213. */
  4214. void __init free_area_init_nodes(unsigned long *max_zone_pfn)
  4215. {
  4216. unsigned long nid;
  4217. int i;
  4218. /* Sort early_node_map as initialisation assumes it is sorted */
  4219. sort_node_map();
  4220. /* Record where the zone boundaries are */
  4221. memset(arch_zone_lowest_possible_pfn, 0,
  4222. sizeof(arch_zone_lowest_possible_pfn));
  4223. memset(arch_zone_highest_possible_pfn, 0,
  4224. sizeof(arch_zone_highest_possible_pfn));
  4225. arch_zone_lowest_possible_pfn[0] = find_min_pfn_with_active_regions();
  4226. arch_zone_highest_possible_pfn[0] = max_zone_pfn[0];
  4227. for (i = 1; i < MAX_NR_ZONES; i++) {
  4228. if (i == ZONE_MOVABLE)
  4229. continue;
  4230. arch_zone_lowest_possible_pfn[i] =
  4231. arch_zone_highest_possible_pfn[i-1];
  4232. arch_zone_highest_possible_pfn[i] =
  4233. max(max_zone_pfn[i], arch_zone_lowest_possible_pfn[i]);
  4234. }
  4235. arch_zone_lowest_possible_pfn[ZONE_MOVABLE] = 0;
  4236. arch_zone_highest_possible_pfn[ZONE_MOVABLE] = 0;
  4237. /* Find the PFNs that ZONE_MOVABLE begins at in each node */
  4238. memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
  4239. find_zone_movable_pfns_for_nodes(zone_movable_pfn);
  4240. /* Print out the zone ranges */
  4241. printk("Zone PFN ranges:\n");
  4242. for (i = 0; i < MAX_NR_ZONES; i++) {
  4243. if (i == ZONE_MOVABLE)
  4244. continue;
  4245. printk(" %-8s ", zone_names[i]);
  4246. if (arch_zone_lowest_possible_pfn[i] ==
  4247. arch_zone_highest_possible_pfn[i])
  4248. printk("empty\n");
  4249. else
  4250. printk("%0#10lx -> %0#10lx\n",
  4251. arch_zone_lowest_possible_pfn[i],
  4252. arch_zone_highest_possible_pfn[i]);
  4253. }
  4254. /* Print out the PFNs ZONE_MOVABLE begins at in each node */
  4255. printk("Movable zone start PFN for each node\n");
  4256. for (i = 0; i < MAX_NUMNODES; i++) {
  4257. if (zone_movable_pfn[i])
  4258. printk(" Node %d: %lu\n", i, zone_movable_pfn[i]);
  4259. }
  4260. /* Print out the early_node_map[] */
  4261. printk("early_node_map[%d] active PFN ranges\n", nr_nodemap_entries);
  4262. for (i = 0; i < nr_nodemap_entries; i++)
  4263. printk(" %3d: %0#10lx -> %0#10lx\n", early_node_map[i].nid,
  4264. early_node_map[i].start_pfn,
  4265. early_node_map[i].end_pfn);
  4266. /* Initialise every node */
  4267. mminit_verify_pageflags_layout();
  4268. setup_nr_node_ids();
  4269. for_each_online_node(nid) {
  4270. pg_data_t *pgdat = NODE_DATA(nid);
  4271. free_area_init_node(nid, NULL,
  4272. find_min_pfn_for_node(nid), NULL);
  4273. /* Any memory on that node */
  4274. if (pgdat->node_present_pages)
  4275. node_set_state(nid, N_HIGH_MEMORY);
  4276. check_for_regular_memory(pgdat);
  4277. }
  4278. }
  4279. static int __init cmdline_parse_core(char *p, unsigned long *core)
  4280. {
  4281. unsigned long long coremem;
  4282. if (!p)
  4283. return -EINVAL;
  4284. coremem = memparse(p, &p);
  4285. *core = coremem >> PAGE_SHIFT;
  4286. /* Paranoid check that UL is enough for the coremem value */
  4287. WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
  4288. return 0;
  4289. }
  4290. /*
  4291. * kernelcore=size sets the amount of memory for use for allocations that
  4292. * cannot be reclaimed or migrated.
  4293. */
  4294. static int __init cmdline_parse_kernelcore(char *p)
  4295. {
  4296. return cmdline_parse_core(p, &required_kernelcore);
  4297. }
  4298. /*
  4299. * movablecore=size sets the amount of memory for use for allocations that
  4300. * can be reclaimed or migrated.
  4301. */
  4302. static int __init cmdline_parse_movablecore(char *p)
  4303. {
  4304. return cmdline_parse_core(p, &required_movablecore);
  4305. }
  4306. early_param("kernelcore", cmdline_parse_kernelcore);
  4307. early_param("movablecore", cmdline_parse_movablecore);
  4308. #endif /* CONFIG_ARCH_POPULATES_NODE_MAP */
  4309. /**
  4310. * set_dma_reserve - set the specified number of pages reserved in the first zone
  4311. * @new_dma_reserve: The number of pages to mark reserved
  4312. *
  4313. * The per-cpu batchsize and zone watermarks are determined by present_pages.
  4314. * In the DMA zone, a significant percentage may be consumed by kernel image
  4315. * and other unfreeable allocations which can skew the watermarks badly. This
  4316. * function may optionally be used to account for unfreeable pages in the
  4317. * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
  4318. * smaller per-cpu batchsize.
  4319. */
  4320. void __init set_dma_reserve(unsigned long new_dma_reserve)
  4321. {
  4322. dma_reserve = new_dma_reserve;
  4323. }
  4324. void __init free_area_init(unsigned long *zones_size)
  4325. {
  4326. free_area_init_node(0, zones_size,
  4327. __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL);
  4328. }
  4329. static int page_alloc_cpu_notify(struct notifier_block *self,
  4330. unsigned long action, void *hcpu)
  4331. {
  4332. int cpu = (unsigned long)hcpu;
  4333. if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
  4334. drain_pages(cpu);
  4335. /*
  4336. * Spill the event counters of the dead processor
  4337. * into the current processors event counters.
  4338. * This artificially elevates the count of the current
  4339. * processor.
  4340. */
  4341. vm_events_fold_cpu(cpu);
  4342. /*
  4343. * Zero the differential counters of the dead processor
  4344. * so that the vm statistics are consistent.
  4345. *
  4346. * This is only okay since the processor is dead and cannot
  4347. * race with what we are doing.
  4348. */
  4349. refresh_cpu_vm_stats(cpu);
  4350. }
  4351. return NOTIFY_OK;
  4352. }
  4353. void __init page_alloc_init(void)
  4354. {
  4355. hotcpu_notifier(page_alloc_cpu_notify, 0);
  4356. }
  4357. /*
  4358. * calculate_totalreserve_pages - called when sysctl_lower_zone_reserve_ratio
  4359. * or min_free_kbytes changes.
  4360. */
  4361. static void calculate_totalreserve_pages(void)
  4362. {
  4363. struct pglist_data *pgdat;
  4364. unsigned long reserve_pages = 0;
  4365. enum zone_type i, j;
  4366. for_each_online_pgdat(pgdat) {
  4367. for (i = 0; i < MAX_NR_ZONES; i++) {
  4368. struct zone *zone = pgdat->node_zones + i;
  4369. unsigned long max = 0;
  4370. /* Find valid and maximum lowmem_reserve in the zone */
  4371. for (j = i; j < MAX_NR_ZONES; j++) {
  4372. if (zone->lowmem_reserve[j] > max)
  4373. max = zone->lowmem_reserve[j];
  4374. }
  4375. /* we treat the high watermark as reserved pages. */
  4376. max += high_wmark_pages(zone);
  4377. if (max > zone->present_pages)
  4378. max = zone->present_pages;
  4379. reserve_pages += max;
  4380. }
  4381. }
  4382. totalreserve_pages = reserve_pages;
  4383. }
  4384. /*
  4385. * setup_per_zone_lowmem_reserve - called whenever
  4386. * sysctl_lower_zone_reserve_ratio changes. Ensures that each zone
  4387. * has a correct pages reserved value, so an adequate number of
  4388. * pages are left in the zone after a successful __alloc_pages().
  4389. */
  4390. static void setup_per_zone_lowmem_reserve(void)
  4391. {
  4392. struct pglist_data *pgdat;
  4393. enum zone_type j, idx;
  4394. for_each_online_pgdat(pgdat) {
  4395. for (j = 0; j < MAX_NR_ZONES; j++) {
  4396. struct zone *zone = pgdat->node_zones + j;
  4397. unsigned long present_pages = zone->present_pages;
  4398. zone->lowmem_reserve[j] = 0;
  4399. idx = j;
  4400. while (idx) {
  4401. struct zone *lower_zone;
  4402. idx--;
  4403. if (sysctl_lowmem_reserve_ratio[idx] < 1)
  4404. sysctl_lowmem_reserve_ratio[idx] = 1;
  4405. lower_zone = pgdat->node_zones + idx;
  4406. lower_zone->lowmem_reserve[j] = present_pages /
  4407. sysctl_lowmem_reserve_ratio[idx];
  4408. present_pages += lower_zone->present_pages;
  4409. }
  4410. }
  4411. }
  4412. /* update totalreserve_pages */
  4413. calculate_totalreserve_pages();
  4414. }
  4415. /**
  4416. * setup_per_zone_wmarks - called when min_free_kbytes changes
  4417. * or when memory is hot-{added|removed}
  4418. *
  4419. * Ensures that the watermark[min,low,high] values for each zone are set
  4420. * correctly with respect to min_free_kbytes.
  4421. */
  4422. void setup_per_zone_wmarks(void)
  4423. {
  4424. unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
  4425. unsigned long lowmem_pages = 0;
  4426. struct zone *zone;
  4427. unsigned long flags;
  4428. /* Calculate total number of !ZONE_HIGHMEM pages */
  4429. for_each_zone(zone) {
  4430. if (!is_highmem(zone))
  4431. lowmem_pages += zone->present_pages;
  4432. }
  4433. for_each_zone(zone) {
  4434. u64 tmp;
  4435. spin_lock_irqsave(&zone->lock, flags);
  4436. tmp = (u64)pages_min * zone->present_pages;
  4437. do_div(tmp, lowmem_pages);
  4438. if (is_highmem(zone)) {
  4439. /*
  4440. * __GFP_HIGH and PF_MEMALLOC allocations usually don't
  4441. * need highmem pages, so cap pages_min to a small
  4442. * value here.
  4443. *
  4444. * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
  4445. * deltas controls asynch page reclaim, and so should
  4446. * not be capped for highmem.
  4447. */
  4448. int min_pages;
  4449. min_pages = zone->present_pages / 1024;
  4450. if (min_pages < SWAP_CLUSTER_MAX)
  4451. min_pages = SWAP_CLUSTER_MAX;
  4452. if (min_pages > 128)
  4453. min_pages = 128;
  4454. zone->watermark[WMARK_MIN] = min_pages;
  4455. } else {
  4456. /*
  4457. * If it's a lowmem zone, reserve a number of pages
  4458. * proportionate to the zone's size.
  4459. */
  4460. zone->watermark[WMARK_MIN] = tmp;
  4461. }
  4462. zone->watermark[WMARK_LOW] = min_wmark_pages(zone) + (tmp >> 2);
  4463. zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + (tmp >> 1);
  4464. setup_zone_migrate_reserve(zone);
  4465. spin_unlock_irqrestore(&zone->lock, flags);
  4466. }
  4467. /* update totalreserve_pages */
  4468. calculate_totalreserve_pages();
  4469. }
  4470. /*
  4471. * The inactive anon list should be small enough that the VM never has to
  4472. * do too much work, but large enough that each inactive page has a chance
  4473. * to be referenced again before it is swapped out.
  4474. *
  4475. * The inactive_anon ratio is the target ratio of ACTIVE_ANON to
  4476. * INACTIVE_ANON pages on this zone's LRU, maintained by the
  4477. * pageout code. A zone->inactive_ratio of 3 means 3:1 or 25% of
  4478. * the anonymous pages are kept on the inactive list.
  4479. *
  4480. * total target max
  4481. * memory ratio inactive anon
  4482. * -------------------------------------
  4483. * 10MB 1 5MB
  4484. * 100MB 1 50MB
  4485. * 1GB 3 250MB
  4486. * 10GB 10 0.9GB
  4487. * 100GB 31 3GB
  4488. * 1TB 101 10GB
  4489. * 10TB 320 32GB
  4490. */
  4491. static void __meminit calculate_zone_inactive_ratio(struct zone *zone)
  4492. {
  4493. unsigned int gb, ratio;
  4494. /* Zone size in gigabytes */
  4495. gb = zone->present_pages >> (30 - PAGE_SHIFT);
  4496. if (gb)
  4497. ratio = int_sqrt(10 * gb);
  4498. else
  4499. ratio = 1;
  4500. zone->inactive_ratio = ratio;
  4501. }
  4502. static void __meminit setup_per_zone_inactive_ratio(void)
  4503. {
  4504. struct zone *zone;
  4505. for_each_zone(zone)
  4506. calculate_zone_inactive_ratio(zone);
  4507. }
  4508. /*
  4509. * Initialise min_free_kbytes.
  4510. *
  4511. * For small machines we want it small (128k min). For large machines
  4512. * we want it large (64MB max). But it is not linear, because network
  4513. * bandwidth does not increase linearly with machine size. We use
  4514. *
  4515. * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy:
  4516. * min_free_kbytes = sqrt(lowmem_kbytes * 16)
  4517. *
  4518. * which yields
  4519. *
  4520. * 16MB: 512k
  4521. * 32MB: 724k
  4522. * 64MB: 1024k
  4523. * 128MB: 1448k
  4524. * 256MB: 2048k
  4525. * 512MB: 2896k
  4526. * 1024MB: 4096k
  4527. * 2048MB: 5792k
  4528. * 4096MB: 8192k
  4529. * 8192MB: 11584k
  4530. * 16384MB: 16384k
  4531. */
  4532. int __meminit init_per_zone_wmark_min(void)
  4533. {
  4534. unsigned long lowmem_kbytes;
  4535. lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
  4536. min_free_kbytes = int_sqrt(lowmem_kbytes * 16);
  4537. if (min_free_kbytes < 128)
  4538. min_free_kbytes = 128;
  4539. if (min_free_kbytes > 65536)
  4540. min_free_kbytes = 65536;
  4541. setup_per_zone_wmarks();
  4542. refresh_zone_stat_thresholds();
  4543. setup_per_zone_lowmem_reserve();
  4544. setup_per_zone_inactive_ratio();
  4545. return 0;
  4546. }
  4547. module_init(init_per_zone_wmark_min)
  4548. /*
  4549. * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so
  4550. * that we can call two helper functions whenever min_free_kbytes
  4551. * changes.
  4552. */
  4553. int min_free_kbytes_sysctl_handler(ctl_table *table, int write,
  4554. void __user *buffer, size_t *length, loff_t *ppos)
  4555. {
  4556. proc_dointvec(table, write, buffer, length, ppos);
  4557. if (write)
  4558. setup_per_zone_wmarks();
  4559. return 0;
  4560. }
  4561. #ifdef CONFIG_NUMA
  4562. int sysctl_min_unmapped_ratio_sysctl_handler(ctl_table *table, int write,
  4563. void __user *buffer, size_t *length, loff_t *ppos)
  4564. {
  4565. struct zone *zone;
  4566. int rc;
  4567. rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
  4568. if (rc)
  4569. return rc;
  4570. for_each_zone(zone)
  4571. zone->min_unmapped_pages = (zone->present_pages *
  4572. sysctl_min_unmapped_ratio) / 100;
  4573. return 0;
  4574. }
  4575. int sysctl_min_slab_ratio_sysctl_handler(ctl_table *table, int write,
  4576. void __user *buffer, size_t *length, loff_t *ppos)
  4577. {
  4578. struct zone *zone;
  4579. int rc;
  4580. rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
  4581. if (rc)
  4582. return rc;
  4583. for_each_zone(zone)
  4584. zone->min_slab_pages = (zone->present_pages *
  4585. sysctl_min_slab_ratio) / 100;
  4586. return 0;
  4587. }
  4588. #endif
  4589. /*
  4590. * lowmem_reserve_ratio_sysctl_handler - just a wrapper around
  4591. * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve()
  4592. * whenever sysctl_lowmem_reserve_ratio changes.
  4593. *
  4594. * The reserve ratio obviously has absolutely no relation with the
  4595. * minimum watermarks. The lowmem reserve ratio can only make sense
  4596. * if in function of the boot time zone sizes.
  4597. */
  4598. int lowmem_reserve_ratio_sysctl_handler(ctl_table *table, int write,
  4599. void __user *buffer, size_t *length, loff_t *ppos)
  4600. {
  4601. proc_dointvec_minmax(table, write, buffer, length, ppos);
  4602. setup_per_zone_lowmem_reserve();
  4603. return 0;
  4604. }
  4605. /*
  4606. * percpu_pagelist_fraction - changes the pcp->high for each zone on each
  4607. * cpu. It is the fraction of total pages in each zone that a hot per cpu pagelist
  4608. * can have before it gets flushed back to buddy allocator.
  4609. */
  4610. int percpu_pagelist_fraction_sysctl_handler(ctl_table *table, int write,
  4611. void __user *buffer, size_t *length, loff_t *ppos)
  4612. {
  4613. struct zone *zone;
  4614. unsigned int cpu;
  4615. int ret;
  4616. ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
  4617. if (!write || (ret == -EINVAL))
  4618. return ret;
  4619. for_each_populated_zone(zone) {
  4620. for_each_possible_cpu(cpu) {
  4621. unsigned long high;
  4622. high = zone->present_pages / percpu_pagelist_fraction;
  4623. setup_pagelist_highmark(
  4624. per_cpu_ptr(zone->pageset, cpu), high);
  4625. }
  4626. }
  4627. return 0;
  4628. }
  4629. int hashdist = HASHDIST_DEFAULT;
  4630. #ifdef CONFIG_NUMA
  4631. static int __init set_hashdist(char *str)
  4632. {
  4633. if (!str)
  4634. return 0;
  4635. hashdist = simple_strtoul(str, &str, 0);
  4636. return 1;
  4637. }
  4638. __setup("hashdist=", set_hashdist);
  4639. #endif
  4640. /*
  4641. * allocate a large system hash table from bootmem
  4642. * - it is assumed that the hash table must contain an exact power-of-2
  4643. * quantity of entries
  4644. * - limit is the number of hash buckets, not the total allocation size
  4645. */
  4646. void *__init alloc_large_system_hash(const char *tablename,
  4647. unsigned long bucketsize,
  4648. unsigned long numentries,
  4649. int scale,
  4650. int flags,
  4651. unsigned int *_hash_shift,
  4652. unsigned int *_hash_mask,
  4653. unsigned long limit)
  4654. {
  4655. unsigned long long max = limit;
  4656. unsigned long log2qty, size;
  4657. void *table = NULL;
  4658. /* allow the kernel cmdline to have a say */
  4659. if (!numentries) {
  4660. /* round applicable memory size up to nearest megabyte */
  4661. numentries = nr_kernel_pages;
  4662. numentries += (1UL << (20 - PAGE_SHIFT)) - 1;
  4663. numentries >>= 20 - PAGE_SHIFT;
  4664. numentries <<= 20 - PAGE_SHIFT;
  4665. /* limit to 1 bucket per 2^scale bytes of low memory */
  4666. if (scale > PAGE_SHIFT)
  4667. numentries >>= (scale - PAGE_SHIFT);
  4668. else
  4669. numentries <<= (PAGE_SHIFT - scale);
  4670. /* Make sure we've got at least a 0-order allocation.. */
  4671. if (unlikely(flags & HASH_SMALL)) {
  4672. /* Makes no sense without HASH_EARLY */
  4673. WARN_ON(!(flags & HASH_EARLY));
  4674. if (!(numentries >> *_hash_shift)) {
  4675. numentries = 1UL << *_hash_shift;
  4676. BUG_ON(!numentries);
  4677. }
  4678. } else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
  4679. numentries = PAGE_SIZE / bucketsize;
  4680. }
  4681. numentries = roundup_pow_of_two(numentries);
  4682. /* limit allocation size to 1/16 total memory by default */
  4683. if (max == 0) {
  4684. max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
  4685. do_div(max, bucketsize);
  4686. }
  4687. if (numentries > max)
  4688. numentries = max;
  4689. log2qty = ilog2(numentries);
  4690. do {
  4691. size = bucketsize << log2qty;
  4692. if (flags & HASH_EARLY)
  4693. table = alloc_bootmem_nopanic(size);
  4694. else if (hashdist)
  4695. table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL);
  4696. else {
  4697. /*
  4698. * If bucketsize is not a power-of-two, we may free
  4699. * some pages at the end of hash table which
  4700. * alloc_pages_exact() automatically does
  4701. */
  4702. if (get_order(size) < MAX_ORDER) {
  4703. table = alloc_pages_exact(size, GFP_ATOMIC);
  4704. kmemleak_alloc(table, size, 1, GFP_ATOMIC);
  4705. }
  4706. }
  4707. } while (!table && size > PAGE_SIZE && --log2qty);
  4708. if (!table)
  4709. panic("Failed to allocate %s hash table\n", tablename);
  4710. printk(KERN_INFO "%s hash table entries: %ld (order: %d, %lu bytes)\n",
  4711. tablename,
  4712. (1UL << log2qty),
  4713. ilog2(size) - PAGE_SHIFT,
  4714. size);
  4715. if (_hash_shift)
  4716. *_hash_shift = log2qty;
  4717. if (_hash_mask)
  4718. *_hash_mask = (1 << log2qty) - 1;
  4719. return table;
  4720. }
  4721. /* Return a pointer to the bitmap storing bits affecting a block of pages */
  4722. static inline unsigned long *get_pageblock_bitmap(struct zone *zone,
  4723. unsigned long pfn)
  4724. {
  4725. #ifdef CONFIG_SPARSEMEM
  4726. return __pfn_to_section(pfn)->pageblock_flags;
  4727. #else
  4728. return zone->pageblock_flags;
  4729. #endif /* CONFIG_SPARSEMEM */
  4730. }
  4731. static inline int pfn_to_bitidx(struct zone *zone, unsigned long pfn)
  4732. {
  4733. #ifdef CONFIG_SPARSEMEM
  4734. pfn &= (PAGES_PER_SECTION-1);
  4735. return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
  4736. #else
  4737. pfn = pfn - zone->zone_start_pfn;
  4738. return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
  4739. #endif /* CONFIG_SPARSEMEM */
  4740. }
  4741. /**
  4742. * get_pageblock_flags_group - Return the requested group of flags for the pageblock_nr_pages block of pages
  4743. * @page: The page within the block of interest
  4744. * @start_bitidx: The first bit of interest to retrieve
  4745. * @end_bitidx: The last bit of interest
  4746. * returns pageblock_bits flags
  4747. */
  4748. unsigned long get_pageblock_flags_group(struct page *page,
  4749. int start_bitidx, int end_bitidx)
  4750. {
  4751. struct zone *zone;
  4752. unsigned long *bitmap;
  4753. unsigned long pfn, bitidx;
  4754. unsigned long flags = 0;
  4755. unsigned long value = 1;
  4756. zone = page_zone(page);
  4757. pfn = page_to_pfn(page);
  4758. bitmap = get_pageblock_bitmap(zone, pfn);
  4759. bitidx = pfn_to_bitidx(zone, pfn);
  4760. for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1)
  4761. if (test_bit(bitidx + start_bitidx, bitmap))
  4762. flags |= value;
  4763. return flags;
  4764. }
  4765. /**
  4766. * set_pageblock_flags_group - Set the requested group of flags for a pageblock_nr_pages block of pages
  4767. * @page: The page within the block of interest
  4768. * @start_bitidx: The first bit of interest
  4769. * @end_bitidx: The last bit of interest
  4770. * @flags: The flags to set
  4771. */
  4772. void set_pageblock_flags_group(struct page *page, unsigned long flags,
  4773. int start_bitidx, int end_bitidx)
  4774. {
  4775. struct zone *zone;
  4776. unsigned long *bitmap;
  4777. unsigned long pfn, bitidx;
  4778. unsigned long value = 1;
  4779. zone = page_zone(page);
  4780. pfn = page_to_pfn(page);
  4781. bitmap = get_pageblock_bitmap(zone, pfn);
  4782. bitidx = pfn_to_bitidx(zone, pfn);
  4783. VM_BUG_ON(pfn < zone->zone_start_pfn);
  4784. VM_BUG_ON(pfn >= zone->zone_start_pfn + zone->spanned_pages);
  4785. for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1)
  4786. if (flags & value)
  4787. __set_bit(bitidx + start_bitidx, bitmap);
  4788. else
  4789. __clear_bit(bitidx + start_bitidx, bitmap);
  4790. }
  4791. /*
  4792. * This is designed as sub function...plz see page_isolation.c also.
  4793. * set/clear page block's type to be ISOLATE.
  4794. * page allocater never alloc memory from ISOLATE block.
  4795. */
  4796. static int
  4797. __count_immobile_pages(struct zone *zone, struct page *page, int count)
  4798. {
  4799. unsigned long pfn, iter, found;
  4800. /*
  4801. * For avoiding noise data, lru_add_drain_all() should be called
  4802. * If ZONE_MOVABLE, the zone never contains immobile pages
  4803. */
  4804. if (zone_idx(zone) == ZONE_MOVABLE)
  4805. return true;
  4806. if (get_pageblock_migratetype(page) == MIGRATE_MOVABLE)
  4807. return true;
  4808. pfn = page_to_pfn(page);
  4809. for (found = 0, iter = 0; iter < pageblock_nr_pages; iter++) {
  4810. unsigned long check = pfn + iter;
  4811. if (!pfn_valid_within(check))
  4812. continue;
  4813. page = pfn_to_page(check);
  4814. if (!page_count(page)) {
  4815. if (PageBuddy(page))
  4816. iter += (1 << page_order(page)) - 1;
  4817. continue;
  4818. }
  4819. if (!PageLRU(page))
  4820. found++;
  4821. /*
  4822. * If there are RECLAIMABLE pages, we need to check it.
  4823. * But now, memory offline itself doesn't call shrink_slab()
  4824. * and it still to be fixed.
  4825. */
  4826. /*
  4827. * If the page is not RAM, page_count()should be 0.
  4828. * we don't need more check. This is an _used_ not-movable page.
  4829. *
  4830. * The problematic thing here is PG_reserved pages. PG_reserved
  4831. * is set to both of a memory hole page and a _used_ kernel
  4832. * page at boot.
  4833. */
  4834. if (found > count)
  4835. return false;
  4836. }
  4837. return true;
  4838. }
  4839. bool is_pageblock_removable_nolock(struct page *page)
  4840. {
  4841. struct zone *zone = page_zone(page);
  4842. return __count_immobile_pages(zone, page, 0);
  4843. }
  4844. int set_migratetype_isolate(struct page *page)
  4845. {
  4846. struct zone *zone;
  4847. unsigned long flags, pfn;
  4848. struct memory_isolate_notify arg;
  4849. int notifier_ret;
  4850. int ret = -EBUSY;
  4851. zone = page_zone(page);
  4852. spin_lock_irqsave(&zone->lock, flags);
  4853. pfn = page_to_pfn(page);
  4854. arg.start_pfn = pfn;
  4855. arg.nr_pages = pageblock_nr_pages;
  4856. arg.pages_found = 0;
  4857. /*
  4858. * It may be possible to isolate a pageblock even if the
  4859. * migratetype is not MIGRATE_MOVABLE. The memory isolation
  4860. * notifier chain is used by balloon drivers to return the
  4861. * number of pages in a range that are held by the balloon
  4862. * driver to shrink memory. If all the pages are accounted for
  4863. * by balloons, are free, or on the LRU, isolation can continue.
  4864. * Later, for example, when memory hotplug notifier runs, these
  4865. * pages reported as "can be isolated" should be isolated(freed)
  4866. * by the balloon driver through the memory notifier chain.
  4867. */
  4868. notifier_ret = memory_isolate_notify(MEM_ISOLATE_COUNT, &arg);
  4869. notifier_ret = notifier_to_errno(notifier_ret);
  4870. if (notifier_ret)
  4871. goto out;
  4872. /*
  4873. * FIXME: Now, memory hotplug doesn't call shrink_slab() by itself.
  4874. * We just check MOVABLE pages.
  4875. */
  4876. if (__count_immobile_pages(zone, page, arg.pages_found))
  4877. ret = 0;
  4878. /*
  4879. * immobile means "not-on-lru" paes. If immobile is larger than
  4880. * removable-by-driver pages reported by notifier, we'll fail.
  4881. */
  4882. out:
  4883. if (!ret) {
  4884. set_pageblock_migratetype(page, MIGRATE_ISOLATE);
  4885. move_freepages_block(zone, page, MIGRATE_ISOLATE);
  4886. }
  4887. spin_unlock_irqrestore(&zone->lock, flags);
  4888. if (!ret)
  4889. drain_all_pages();
  4890. return ret;
  4891. }
  4892. void unset_migratetype_isolate(struct page *page)
  4893. {
  4894. struct zone *zone;
  4895. unsigned long flags;
  4896. zone = page_zone(page);
  4897. spin_lock_irqsave(&zone->lock, flags);
  4898. if (get_pageblock_migratetype(page) != MIGRATE_ISOLATE)
  4899. goto out;
  4900. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  4901. move_freepages_block(zone, page, MIGRATE_MOVABLE);
  4902. out:
  4903. spin_unlock_irqrestore(&zone->lock, flags);
  4904. }
  4905. #ifdef CONFIG_MEMORY_HOTREMOVE
  4906. /*
  4907. * All pages in the range must be isolated before calling this.
  4908. */
  4909. void
  4910. __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
  4911. {
  4912. struct page *page;
  4913. struct zone *zone;
  4914. int order, i;
  4915. unsigned long pfn;
  4916. unsigned long flags;
  4917. /* find the first valid pfn */
  4918. for (pfn = start_pfn; pfn < end_pfn; pfn++)
  4919. if (pfn_valid(pfn))
  4920. break;
  4921. if (pfn == end_pfn)
  4922. return;
  4923. zone = page_zone(pfn_to_page(pfn));
  4924. spin_lock_irqsave(&zone->lock, flags);
  4925. pfn = start_pfn;
  4926. while (pfn < end_pfn) {
  4927. if (!pfn_valid(pfn)) {
  4928. pfn++;
  4929. continue;
  4930. }
  4931. page = pfn_to_page(pfn);
  4932. BUG_ON(page_count(page));
  4933. BUG_ON(!PageBuddy(page));
  4934. order = page_order(page);
  4935. #ifdef CONFIG_DEBUG_VM
  4936. printk(KERN_INFO "remove from free list %lx %d %lx\n",
  4937. pfn, 1 << order, end_pfn);
  4938. #endif
  4939. list_del(&page->lru);
  4940. rmv_page_order(page);
  4941. zone->free_area[order].nr_free--;
  4942. __mod_zone_page_state(zone, NR_FREE_PAGES,
  4943. - (1UL << order));
  4944. for (i = 0; i < (1 << order); i++)
  4945. SetPageReserved((page+i));
  4946. pfn += (1 << order);
  4947. }
  4948. spin_unlock_irqrestore(&zone->lock, flags);
  4949. }
  4950. #endif
  4951. #ifdef CONFIG_MEMORY_FAILURE
  4952. bool is_free_buddy_page(struct page *page)
  4953. {
  4954. struct zone *zone = page_zone(page);
  4955. unsigned long pfn = page_to_pfn(page);
  4956. unsigned long flags;
  4957. int order;
  4958. spin_lock_irqsave(&zone->lock, flags);
  4959. for (order = 0; order < MAX_ORDER; order++) {
  4960. struct page *page_head = page - (pfn & ((1 << order) - 1));
  4961. if (PageBuddy(page_head) && page_order(page_head) >= order)
  4962. break;
  4963. }
  4964. spin_unlock_irqrestore(&zone->lock, flags);
  4965. return order < MAX_ORDER;
  4966. }
  4967. #endif
  4968. static struct trace_print_flags pageflag_names[] = {
  4969. {1UL << PG_locked, "locked" },
  4970. {1UL << PG_error, "error" },
  4971. {1UL << PG_referenced, "referenced" },
  4972. {1UL << PG_uptodate, "uptodate" },
  4973. {1UL << PG_dirty, "dirty" },
  4974. {1UL << PG_lru, "lru" },
  4975. {1UL << PG_active, "active" },
  4976. {1UL << PG_slab, "slab" },
  4977. {1UL << PG_owner_priv_1, "owner_priv_1" },
  4978. {1UL << PG_arch_1, "arch_1" },
  4979. {1UL << PG_reserved, "reserved" },
  4980. {1UL << PG_private, "private" },
  4981. {1UL << PG_private_2, "private_2" },
  4982. {1UL << PG_writeback, "writeback" },
  4983. #ifdef CONFIG_PAGEFLAGS_EXTENDED
  4984. {1UL << PG_head, "head" },
  4985. {1UL << PG_tail, "tail" },
  4986. #else
  4987. {1UL << PG_compound, "compound" },
  4988. #endif
  4989. {1UL << PG_swapcache, "swapcache" },
  4990. {1UL << PG_mappedtodisk, "mappedtodisk" },
  4991. {1UL << PG_reclaim, "reclaim" },
  4992. {1UL << PG_swapbacked, "swapbacked" },
  4993. {1UL << PG_unevictable, "unevictable" },
  4994. #ifdef CONFIG_MMU
  4995. {1UL << PG_mlocked, "mlocked" },
  4996. #endif
  4997. #ifdef CONFIG_ARCH_USES_PG_UNCACHED
  4998. {1UL << PG_uncached, "uncached" },
  4999. #endif
  5000. #ifdef CONFIG_MEMORY_FAILURE
  5001. {1UL << PG_hwpoison, "hwpoison" },
  5002. #endif
  5003. {-1UL, NULL },
  5004. };
  5005. static void dump_page_flags(unsigned long flags)
  5006. {
  5007. const char *delim = "";
  5008. unsigned long mask;
  5009. int i;
  5010. printk(KERN_ALERT "page flags: %#lx(", flags);
  5011. /* remove zone id */
  5012. flags &= (1UL << NR_PAGEFLAGS) - 1;
  5013. for (i = 0; pageflag_names[i].name && flags; i++) {
  5014. mask = pageflag_names[i].mask;
  5015. if ((flags & mask) != mask)
  5016. continue;
  5017. flags &= ~mask;
  5018. printk("%s%s", delim, pageflag_names[i].name);
  5019. delim = "|";
  5020. }
  5021. /* check for left over flags */
  5022. if (flags)
  5023. printk("%s%#lx", delim, flags);
  5024. printk(")\n");
  5025. }
  5026. void dump_page(struct page *page)
  5027. {
  5028. printk(KERN_ALERT
  5029. "page:%p count:%d mapcount:%d mapping:%p index:%#lx\n",
  5030. page, atomic_read(&page->_count), page_mapcount(page),
  5031. page->mapping, page->index);
  5032. dump_page_flags(page->flags);
  5033. mem_cgroup_print_bad_page(page);
  5034. }