page_alloc.c 142 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185
  1. /*
  2. * linux/mm/page_alloc.c
  3. *
  4. * Manages the free list, the system allocates free pages here.
  5. * Note that kmalloc() lives in slab.c
  6. *
  7. * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
  8. * Swap reorganised 29.12.95, Stephen Tweedie
  9. * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
  10. * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
  11. * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
  12. * Zone balancing, Kanoj Sarcar, SGI, Jan 2000
  13. * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
  14. * (lots of bits borrowed from Ingo Molnar & Andrew Morton)
  15. */
  16. #include <linux/stddef.h>
  17. #include <linux/mm.h>
  18. #include <linux/swap.h>
  19. #include <linux/interrupt.h>
  20. #include <linux/pagemap.h>
  21. #include <linux/jiffies.h>
  22. #include <linux/bootmem.h>
  23. #include <linux/compiler.h>
  24. #include <linux/kernel.h>
  25. #include <linux/kmemcheck.h>
  26. #include <linux/module.h>
  27. #include <linux/suspend.h>
  28. #include <linux/pagevec.h>
  29. #include <linux/blkdev.h>
  30. #include <linux/slab.h>
  31. #include <linux/oom.h>
  32. #include <linux/notifier.h>
  33. #include <linux/topology.h>
  34. #include <linux/sysctl.h>
  35. #include <linux/cpu.h>
  36. #include <linux/cpuset.h>
  37. #include <linux/memory_hotplug.h>
  38. #include <linux/nodemask.h>
  39. #include <linux/vmalloc.h>
  40. #include <linux/mempolicy.h>
  41. #include <linux/stop_machine.h>
  42. #include <linux/sort.h>
  43. #include <linux/pfn.h>
  44. #include <linux/backing-dev.h>
  45. #include <linux/fault-inject.h>
  46. #include <linux/page-isolation.h>
  47. #include <linux/page_cgroup.h>
  48. #include <linux/debugobjects.h>
  49. #include <linux/kmemleak.h>
  50. #include <linux/memory.h>
  51. #include <trace/events/kmem.h>
  52. #include <asm/tlbflush.h>
  53. #include <asm/div64.h>
  54. #include "internal.h"
  55. /*
  56. * Array of node states.
  57. */
  58. nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
  59. [N_POSSIBLE] = NODE_MASK_ALL,
  60. [N_ONLINE] = { { [0] = 1UL } },
  61. #ifndef CONFIG_NUMA
  62. [N_NORMAL_MEMORY] = { { [0] = 1UL } },
  63. #ifdef CONFIG_HIGHMEM
  64. [N_HIGH_MEMORY] = { { [0] = 1UL } },
  65. #endif
  66. [N_CPU] = { { [0] = 1UL } },
  67. #endif /* NUMA */
  68. };
  69. EXPORT_SYMBOL(node_states);
  70. unsigned long totalram_pages __read_mostly;
  71. unsigned long totalreserve_pages __read_mostly;
  72. int percpu_pagelist_fraction;
  73. gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
  74. #ifdef CONFIG_PM_SLEEP
  75. /*
  76. * The following functions are used by the suspend/hibernate code to temporarily
  77. * change gfp_allowed_mask in order to avoid using I/O during memory allocations
  78. * while devices are suspended. To avoid races with the suspend/hibernate code,
  79. * they should always be called with pm_mutex held (gfp_allowed_mask also should
  80. * only be modified with pm_mutex held, unless the suspend/hibernate code is
  81. * guaranteed not to run in parallel with that modification).
  82. */
  83. void set_gfp_allowed_mask(gfp_t mask)
  84. {
  85. WARN_ON(!mutex_is_locked(&pm_mutex));
  86. gfp_allowed_mask = mask;
  87. }
  88. gfp_t clear_gfp_allowed_mask(gfp_t mask)
  89. {
  90. gfp_t ret = gfp_allowed_mask;
  91. WARN_ON(!mutex_is_locked(&pm_mutex));
  92. gfp_allowed_mask &= ~mask;
  93. return ret;
  94. }
  95. #endif /* CONFIG_PM_SLEEP */
  96. #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
  97. int pageblock_order __read_mostly;
  98. #endif
  99. static void __free_pages_ok(struct page *page, unsigned int order);
  100. /*
  101. * results with 256, 32 in the lowmem_reserve sysctl:
  102. * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
  103. * 1G machine -> (16M dma, 784M normal, 224M high)
  104. * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
  105. * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
  106. * HIGHMEM allocation will (224M+784M)/256 of ram reserved in ZONE_DMA
  107. *
  108. * TBD: should special case ZONE_DMA32 machines here - in those we normally
  109. * don't need any ZONE_NORMAL reservation
  110. */
  111. int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = {
  112. #ifdef CONFIG_ZONE_DMA
  113. 256,
  114. #endif
  115. #ifdef CONFIG_ZONE_DMA32
  116. 256,
  117. #endif
  118. #ifdef CONFIG_HIGHMEM
  119. 32,
  120. #endif
  121. 32,
  122. };
  123. EXPORT_SYMBOL(totalram_pages);
  124. static char * const zone_names[MAX_NR_ZONES] = {
  125. #ifdef CONFIG_ZONE_DMA
  126. "DMA",
  127. #endif
  128. #ifdef CONFIG_ZONE_DMA32
  129. "DMA32",
  130. #endif
  131. "Normal",
  132. #ifdef CONFIG_HIGHMEM
  133. "HighMem",
  134. #endif
  135. "Movable",
  136. };
  137. int min_free_kbytes = 1024;
  138. static unsigned long __meminitdata nr_kernel_pages;
  139. static unsigned long __meminitdata nr_all_pages;
  140. static unsigned long __meminitdata dma_reserve;
  141. #ifdef CONFIG_ARCH_POPULATES_NODE_MAP
  142. /*
  143. * MAX_ACTIVE_REGIONS determines the maximum number of distinct
  144. * ranges of memory (RAM) that may be registered with add_active_range().
  145. * Ranges passed to add_active_range() will be merged if possible
  146. * so the number of times add_active_range() can be called is
  147. * related to the number of nodes and the number of holes
  148. */
  149. #ifdef CONFIG_MAX_ACTIVE_REGIONS
  150. /* Allow an architecture to set MAX_ACTIVE_REGIONS to save memory */
  151. #define MAX_ACTIVE_REGIONS CONFIG_MAX_ACTIVE_REGIONS
  152. #else
  153. #if MAX_NUMNODES >= 32
  154. /* If there can be many nodes, allow up to 50 holes per node */
  155. #define MAX_ACTIVE_REGIONS (MAX_NUMNODES*50)
  156. #else
  157. /* By default, allow up to 256 distinct regions */
  158. #define MAX_ACTIVE_REGIONS 256
  159. #endif
  160. #endif
  161. static struct node_active_region __meminitdata early_node_map[MAX_ACTIVE_REGIONS];
  162. static int __meminitdata nr_nodemap_entries;
  163. static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES];
  164. static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES];
  165. static unsigned long __initdata required_kernelcore;
  166. static unsigned long __initdata required_movablecore;
  167. static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES];
  168. /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
  169. int movable_zone;
  170. EXPORT_SYMBOL(movable_zone);
  171. #endif /* CONFIG_ARCH_POPULATES_NODE_MAP */
  172. #if MAX_NUMNODES > 1
  173. int nr_node_ids __read_mostly = MAX_NUMNODES;
  174. int nr_online_nodes __read_mostly = 1;
  175. EXPORT_SYMBOL(nr_node_ids);
  176. EXPORT_SYMBOL(nr_online_nodes);
  177. #endif
  178. int page_group_by_mobility_disabled __read_mostly;
  179. static void set_pageblock_migratetype(struct page *page, int migratetype)
  180. {
  181. if (unlikely(page_group_by_mobility_disabled))
  182. migratetype = MIGRATE_UNMOVABLE;
  183. set_pageblock_flags_group(page, (unsigned long)migratetype,
  184. PB_migrate, PB_migrate_end);
  185. }
  186. bool oom_killer_disabled __read_mostly;
  187. #ifdef CONFIG_DEBUG_VM
  188. static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
  189. {
  190. int ret = 0;
  191. unsigned seq;
  192. unsigned long pfn = page_to_pfn(page);
  193. do {
  194. seq = zone_span_seqbegin(zone);
  195. if (pfn >= zone->zone_start_pfn + zone->spanned_pages)
  196. ret = 1;
  197. else if (pfn < zone->zone_start_pfn)
  198. ret = 1;
  199. } while (zone_span_seqretry(zone, seq));
  200. return ret;
  201. }
  202. static int page_is_consistent(struct zone *zone, struct page *page)
  203. {
  204. if (!pfn_valid_within(page_to_pfn(page)))
  205. return 0;
  206. if (zone != page_zone(page))
  207. return 0;
  208. return 1;
  209. }
  210. /*
  211. * Temporary debugging check for pages not lying within a given zone.
  212. */
  213. static int bad_range(struct zone *zone, struct page *page)
  214. {
  215. if (page_outside_zone_boundaries(zone, page))
  216. return 1;
  217. if (!page_is_consistent(zone, page))
  218. return 1;
  219. return 0;
  220. }
  221. #else
  222. static inline int bad_range(struct zone *zone, struct page *page)
  223. {
  224. return 0;
  225. }
  226. #endif
  227. static void bad_page(struct page *page)
  228. {
  229. static unsigned long resume;
  230. static unsigned long nr_shown;
  231. static unsigned long nr_unshown;
  232. /* Don't complain about poisoned pages */
  233. if (PageHWPoison(page)) {
  234. __ClearPageBuddy(page);
  235. return;
  236. }
  237. /*
  238. * Allow a burst of 60 reports, then keep quiet for that minute;
  239. * or allow a steady drip of one report per second.
  240. */
  241. if (nr_shown == 60) {
  242. if (time_before(jiffies, resume)) {
  243. nr_unshown++;
  244. goto out;
  245. }
  246. if (nr_unshown) {
  247. printk(KERN_ALERT
  248. "BUG: Bad page state: %lu messages suppressed\n",
  249. nr_unshown);
  250. nr_unshown = 0;
  251. }
  252. nr_shown = 0;
  253. }
  254. if (nr_shown++ == 0)
  255. resume = jiffies + 60 * HZ;
  256. printk(KERN_ALERT "BUG: Bad page state in process %s pfn:%05lx\n",
  257. current->comm, page_to_pfn(page));
  258. printk(KERN_ALERT
  259. "page:%p flags:%p count:%d mapcount:%d mapping:%p index:%lx\n",
  260. page, (void *)page->flags, page_count(page),
  261. page_mapcount(page), page->mapping, page->index);
  262. dump_stack();
  263. out:
  264. /* Leave bad fields for debug, except PageBuddy could make trouble */
  265. __ClearPageBuddy(page);
  266. add_taint(TAINT_BAD_PAGE);
  267. }
  268. /*
  269. * Higher-order pages are called "compound pages". They are structured thusly:
  270. *
  271. * The first PAGE_SIZE page is called the "head page".
  272. *
  273. * The remaining PAGE_SIZE pages are called "tail pages".
  274. *
  275. * All pages have PG_compound set. All pages have their ->private pointing at
  276. * the head page (even the head page has this).
  277. *
  278. * The first tail page's ->lru.next holds the address of the compound page's
  279. * put_page() function. Its ->lru.prev holds the order of allocation.
  280. * This usage means that zero-order pages may not be compound.
  281. */
  282. static void free_compound_page(struct page *page)
  283. {
  284. __free_pages_ok(page, compound_order(page));
  285. }
  286. void prep_compound_page(struct page *page, unsigned long order)
  287. {
  288. int i;
  289. int nr_pages = 1 << order;
  290. set_compound_page_dtor(page, free_compound_page);
  291. set_compound_order(page, order);
  292. __SetPageHead(page);
  293. for (i = 1; i < nr_pages; i++) {
  294. struct page *p = page + i;
  295. __SetPageTail(p);
  296. p->first_page = page;
  297. }
  298. }
  299. static int destroy_compound_page(struct page *page, unsigned long order)
  300. {
  301. int i;
  302. int nr_pages = 1 << order;
  303. int bad = 0;
  304. if (unlikely(compound_order(page) != order) ||
  305. unlikely(!PageHead(page))) {
  306. bad_page(page);
  307. bad++;
  308. }
  309. __ClearPageHead(page);
  310. for (i = 1; i < nr_pages; i++) {
  311. struct page *p = page + i;
  312. if (unlikely(!PageTail(p) || (p->first_page != page))) {
  313. bad_page(page);
  314. bad++;
  315. }
  316. __ClearPageTail(p);
  317. }
  318. return bad;
  319. }
  320. static inline void prep_zero_page(struct page *page, int order, gfp_t gfp_flags)
  321. {
  322. int i;
  323. /*
  324. * clear_highpage() will use KM_USER0, so it's a bug to use __GFP_ZERO
  325. * and __GFP_HIGHMEM from hard or soft interrupt context.
  326. */
  327. VM_BUG_ON((gfp_flags & __GFP_HIGHMEM) && in_interrupt());
  328. for (i = 0; i < (1 << order); i++)
  329. clear_highpage(page + i);
  330. }
  331. static inline void set_page_order(struct page *page, int order)
  332. {
  333. set_page_private(page, order);
  334. __SetPageBuddy(page);
  335. }
  336. static inline void rmv_page_order(struct page *page)
  337. {
  338. __ClearPageBuddy(page);
  339. set_page_private(page, 0);
  340. }
  341. /*
  342. * Locate the struct page for both the matching buddy in our
  343. * pair (buddy1) and the combined O(n+1) page they form (page).
  344. *
  345. * 1) Any buddy B1 will have an order O twin B2 which satisfies
  346. * the following equation:
  347. * B2 = B1 ^ (1 << O)
  348. * For example, if the starting buddy (buddy2) is #8 its order
  349. * 1 buddy is #10:
  350. * B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10
  351. *
  352. * 2) Any buddy B will have an order O+1 parent P which
  353. * satisfies the following equation:
  354. * P = B & ~(1 << O)
  355. *
  356. * Assumption: *_mem_map is contiguous at least up to MAX_ORDER
  357. */
  358. static inline struct page *
  359. __page_find_buddy(struct page *page, unsigned long page_idx, unsigned int order)
  360. {
  361. unsigned long buddy_idx = page_idx ^ (1 << order);
  362. return page + (buddy_idx - page_idx);
  363. }
  364. static inline unsigned long
  365. __find_combined_index(unsigned long page_idx, unsigned int order)
  366. {
  367. return (page_idx & ~(1 << order));
  368. }
  369. /*
  370. * This function checks whether a page is free && is the buddy
  371. * we can do coalesce a page and its buddy if
  372. * (a) the buddy is not in a hole &&
  373. * (b) the buddy is in the buddy system &&
  374. * (c) a page and its buddy have the same order &&
  375. * (d) a page and its buddy are in the same zone.
  376. *
  377. * For recording whether a page is in the buddy system, we use PG_buddy.
  378. * Setting, clearing, and testing PG_buddy is serialized by zone->lock.
  379. *
  380. * For recording page's order, we use page_private(page).
  381. */
  382. static inline int page_is_buddy(struct page *page, struct page *buddy,
  383. int order)
  384. {
  385. if (!pfn_valid_within(page_to_pfn(buddy)))
  386. return 0;
  387. if (page_zone_id(page) != page_zone_id(buddy))
  388. return 0;
  389. if (PageBuddy(buddy) && page_order(buddy) == order) {
  390. VM_BUG_ON(page_count(buddy) != 0);
  391. return 1;
  392. }
  393. return 0;
  394. }
  395. /*
  396. * Freeing function for a buddy system allocator.
  397. *
  398. * The concept of a buddy system is to maintain direct-mapped table
  399. * (containing bit values) for memory blocks of various "orders".
  400. * The bottom level table contains the map for the smallest allocatable
  401. * units of memory (here, pages), and each level above it describes
  402. * pairs of units from the levels below, hence, "buddies".
  403. * At a high level, all that happens here is marking the table entry
  404. * at the bottom level available, and propagating the changes upward
  405. * as necessary, plus some accounting needed to play nicely with other
  406. * parts of the VM system.
  407. * At each level, we keep a list of pages, which are heads of continuous
  408. * free pages of length of (1 << order) and marked with PG_buddy. Page's
  409. * order is recorded in page_private(page) field.
  410. * So when we are allocating or freeing one, we can derive the state of the
  411. * other. That is, if we allocate a small block, and both were
  412. * free, the remainder of the region must be split into blocks.
  413. * If a block is freed, and its buddy is also free, then this
  414. * triggers coalescing into a block of larger size.
  415. *
  416. * -- wli
  417. */
  418. static inline void __free_one_page(struct page *page,
  419. struct zone *zone, unsigned int order,
  420. int migratetype)
  421. {
  422. unsigned long page_idx;
  423. if (unlikely(PageCompound(page)))
  424. if (unlikely(destroy_compound_page(page, order)))
  425. return;
  426. VM_BUG_ON(migratetype == -1);
  427. page_idx = page_to_pfn(page) & ((1 << MAX_ORDER) - 1);
  428. VM_BUG_ON(page_idx & ((1 << order) - 1));
  429. VM_BUG_ON(bad_range(zone, page));
  430. while (order < MAX_ORDER-1) {
  431. unsigned long combined_idx;
  432. struct page *buddy;
  433. buddy = __page_find_buddy(page, page_idx, order);
  434. if (!page_is_buddy(page, buddy, order))
  435. break;
  436. /* Our buddy is free, merge with it and move up one order. */
  437. list_del(&buddy->lru);
  438. zone->free_area[order].nr_free--;
  439. rmv_page_order(buddy);
  440. combined_idx = __find_combined_index(page_idx, order);
  441. page = page + (combined_idx - page_idx);
  442. page_idx = combined_idx;
  443. order++;
  444. }
  445. set_page_order(page, order);
  446. list_add(&page->lru,
  447. &zone->free_area[order].free_list[migratetype]);
  448. zone->free_area[order].nr_free++;
  449. }
  450. /*
  451. * free_page_mlock() -- clean up attempts to free and mlocked() page.
  452. * Page should not be on lru, so no need to fix that up.
  453. * free_pages_check() will verify...
  454. */
  455. static inline void free_page_mlock(struct page *page)
  456. {
  457. __dec_zone_page_state(page, NR_MLOCK);
  458. __count_vm_event(UNEVICTABLE_MLOCKFREED);
  459. }
  460. static inline int free_pages_check(struct page *page)
  461. {
  462. if (unlikely(page_mapcount(page) |
  463. (page->mapping != NULL) |
  464. (atomic_read(&page->_count) != 0) |
  465. (page->flags & PAGE_FLAGS_CHECK_AT_FREE))) {
  466. bad_page(page);
  467. return 1;
  468. }
  469. if (page->flags & PAGE_FLAGS_CHECK_AT_PREP)
  470. page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
  471. return 0;
  472. }
  473. /*
  474. * Frees a number of pages from the PCP lists
  475. * Assumes all pages on list are in same zone, and of same order.
  476. * count is the number of pages to free.
  477. *
  478. * If the zone was previously in an "all pages pinned" state then look to
  479. * see if this freeing clears that state.
  480. *
  481. * And clear the zone's pages_scanned counter, to hold off the "all pages are
  482. * pinned" detection logic.
  483. */
  484. static void free_pcppages_bulk(struct zone *zone, int count,
  485. struct per_cpu_pages *pcp)
  486. {
  487. int migratetype = 0;
  488. int batch_free = 0;
  489. spin_lock(&zone->lock);
  490. zone->all_unreclaimable = 0;
  491. zone->pages_scanned = 0;
  492. __mod_zone_page_state(zone, NR_FREE_PAGES, count);
  493. while (count) {
  494. struct page *page;
  495. struct list_head *list;
  496. /*
  497. * Remove pages from lists in a round-robin fashion. A
  498. * batch_free count is maintained that is incremented when an
  499. * empty list is encountered. This is so more pages are freed
  500. * off fuller lists instead of spinning excessively around empty
  501. * lists
  502. */
  503. do {
  504. batch_free++;
  505. if (++migratetype == MIGRATE_PCPTYPES)
  506. migratetype = 0;
  507. list = &pcp->lists[migratetype];
  508. } while (list_empty(list));
  509. do {
  510. page = list_entry(list->prev, struct page, lru);
  511. /* must delete as __free_one_page list manipulates */
  512. list_del(&page->lru);
  513. /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */
  514. __free_one_page(page, zone, 0, page_private(page));
  515. trace_mm_page_pcpu_drain(page, 0, page_private(page));
  516. } while (--count && --batch_free && !list_empty(list));
  517. }
  518. spin_unlock(&zone->lock);
  519. }
  520. static void free_one_page(struct zone *zone, struct page *page, int order,
  521. int migratetype)
  522. {
  523. spin_lock(&zone->lock);
  524. zone->all_unreclaimable = 0;
  525. zone->pages_scanned = 0;
  526. __mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order);
  527. __free_one_page(page, zone, order, migratetype);
  528. spin_unlock(&zone->lock);
  529. }
  530. static void __free_pages_ok(struct page *page, unsigned int order)
  531. {
  532. unsigned long flags;
  533. int i;
  534. int bad = 0;
  535. int wasMlocked = __TestClearPageMlocked(page);
  536. trace_mm_page_free_direct(page, order);
  537. kmemcheck_free_shadow(page, order);
  538. for (i = 0 ; i < (1 << order) ; ++i)
  539. bad += free_pages_check(page + i);
  540. if (bad)
  541. return;
  542. if (!PageHighMem(page)) {
  543. debug_check_no_locks_freed(page_address(page),PAGE_SIZE<<order);
  544. debug_check_no_obj_freed(page_address(page),
  545. PAGE_SIZE << order);
  546. }
  547. arch_free_page(page, order);
  548. kernel_map_pages(page, 1 << order, 0);
  549. local_irq_save(flags);
  550. if (unlikely(wasMlocked))
  551. free_page_mlock(page);
  552. __count_vm_events(PGFREE, 1 << order);
  553. free_one_page(page_zone(page), page, order,
  554. get_pageblock_migratetype(page));
  555. local_irq_restore(flags);
  556. }
  557. /*
  558. * permit the bootmem allocator to evade page validation on high-order frees
  559. */
  560. void __meminit __free_pages_bootmem(struct page *page, unsigned int order)
  561. {
  562. if (order == 0) {
  563. __ClearPageReserved(page);
  564. set_page_count(page, 0);
  565. set_page_refcounted(page);
  566. __free_page(page);
  567. } else {
  568. int loop;
  569. prefetchw(page);
  570. for (loop = 0; loop < BITS_PER_LONG; loop++) {
  571. struct page *p = &page[loop];
  572. if (loop + 1 < BITS_PER_LONG)
  573. prefetchw(p + 1);
  574. __ClearPageReserved(p);
  575. set_page_count(p, 0);
  576. }
  577. set_page_refcounted(page);
  578. __free_pages(page, order);
  579. }
  580. }
  581. /*
  582. * The order of subdivision here is critical for the IO subsystem.
  583. * Please do not alter this order without good reasons and regression
  584. * testing. Specifically, as large blocks of memory are subdivided,
  585. * the order in which smaller blocks are delivered depends on the order
  586. * they're subdivided in this function. This is the primary factor
  587. * influencing the order in which pages are delivered to the IO
  588. * subsystem according to empirical testing, and this is also justified
  589. * by considering the behavior of a buddy system containing a single
  590. * large block of memory acted on by a series of small allocations.
  591. * This behavior is a critical factor in sglist merging's success.
  592. *
  593. * -- wli
  594. */
  595. static inline void expand(struct zone *zone, struct page *page,
  596. int low, int high, struct free_area *area,
  597. int migratetype)
  598. {
  599. unsigned long size = 1 << high;
  600. while (high > low) {
  601. area--;
  602. high--;
  603. size >>= 1;
  604. VM_BUG_ON(bad_range(zone, &page[size]));
  605. list_add(&page[size].lru, &area->free_list[migratetype]);
  606. area->nr_free++;
  607. set_page_order(&page[size], high);
  608. }
  609. }
  610. /*
  611. * This page is about to be returned from the page allocator
  612. */
  613. static inline int check_new_page(struct page *page)
  614. {
  615. if (unlikely(page_mapcount(page) |
  616. (page->mapping != NULL) |
  617. (atomic_read(&page->_count) != 0) |
  618. (page->flags & PAGE_FLAGS_CHECK_AT_PREP))) {
  619. bad_page(page);
  620. return 1;
  621. }
  622. return 0;
  623. }
  624. static int prep_new_page(struct page *page, int order, gfp_t gfp_flags)
  625. {
  626. int i;
  627. for (i = 0; i < (1 << order); i++) {
  628. struct page *p = page + i;
  629. if (unlikely(check_new_page(p)))
  630. return 1;
  631. }
  632. set_page_private(page, 0);
  633. set_page_refcounted(page);
  634. arch_alloc_page(page, order);
  635. kernel_map_pages(page, 1 << order, 1);
  636. if (gfp_flags & __GFP_ZERO)
  637. prep_zero_page(page, order, gfp_flags);
  638. if (order && (gfp_flags & __GFP_COMP))
  639. prep_compound_page(page, order);
  640. return 0;
  641. }
  642. /*
  643. * Go through the free lists for the given migratetype and remove
  644. * the smallest available page from the freelists
  645. */
  646. static inline
  647. struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
  648. int migratetype)
  649. {
  650. unsigned int current_order;
  651. struct free_area * area;
  652. struct page *page;
  653. /* Find a page of the appropriate size in the preferred list */
  654. for (current_order = order; current_order < MAX_ORDER; ++current_order) {
  655. area = &(zone->free_area[current_order]);
  656. if (list_empty(&area->free_list[migratetype]))
  657. continue;
  658. page = list_entry(area->free_list[migratetype].next,
  659. struct page, lru);
  660. list_del(&page->lru);
  661. rmv_page_order(page);
  662. area->nr_free--;
  663. expand(zone, page, order, current_order, area, migratetype);
  664. return page;
  665. }
  666. return NULL;
  667. }
  668. /*
  669. * This array describes the order lists are fallen back to when
  670. * the free lists for the desirable migrate type are depleted
  671. */
  672. static int fallbacks[MIGRATE_TYPES][MIGRATE_TYPES-1] = {
  673. [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE },
  674. [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE },
  675. [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },
  676. [MIGRATE_RESERVE] = { MIGRATE_RESERVE, MIGRATE_RESERVE, MIGRATE_RESERVE }, /* Never used */
  677. };
  678. /*
  679. * Move the free pages in a range to the free lists of the requested type.
  680. * Note that start_page and end_pages are not aligned on a pageblock
  681. * boundary. If alignment is required, use move_freepages_block()
  682. */
  683. static int move_freepages(struct zone *zone,
  684. struct page *start_page, struct page *end_page,
  685. int migratetype)
  686. {
  687. struct page *page;
  688. unsigned long order;
  689. int pages_moved = 0;
  690. #ifndef CONFIG_HOLES_IN_ZONE
  691. /*
  692. * page_zone is not safe to call in this context when
  693. * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant
  694. * anyway as we check zone boundaries in move_freepages_block().
  695. * Remove at a later date when no bug reports exist related to
  696. * grouping pages by mobility
  697. */
  698. BUG_ON(page_zone(start_page) != page_zone(end_page));
  699. #endif
  700. for (page = start_page; page <= end_page;) {
  701. /* Make sure we are not inadvertently changing nodes */
  702. VM_BUG_ON(page_to_nid(page) != zone_to_nid(zone));
  703. if (!pfn_valid_within(page_to_pfn(page))) {
  704. page++;
  705. continue;
  706. }
  707. if (!PageBuddy(page)) {
  708. page++;
  709. continue;
  710. }
  711. order = page_order(page);
  712. list_del(&page->lru);
  713. list_add(&page->lru,
  714. &zone->free_area[order].free_list[migratetype]);
  715. page += 1 << order;
  716. pages_moved += 1 << order;
  717. }
  718. return pages_moved;
  719. }
  720. static int move_freepages_block(struct zone *zone, struct page *page,
  721. int migratetype)
  722. {
  723. unsigned long start_pfn, end_pfn;
  724. struct page *start_page, *end_page;
  725. start_pfn = page_to_pfn(page);
  726. start_pfn = start_pfn & ~(pageblock_nr_pages-1);
  727. start_page = pfn_to_page(start_pfn);
  728. end_page = start_page + pageblock_nr_pages - 1;
  729. end_pfn = start_pfn + pageblock_nr_pages - 1;
  730. /* Do not cross zone boundaries */
  731. if (start_pfn < zone->zone_start_pfn)
  732. start_page = page;
  733. if (end_pfn >= zone->zone_start_pfn + zone->spanned_pages)
  734. return 0;
  735. return move_freepages(zone, start_page, end_page, migratetype);
  736. }
  737. static void change_pageblock_range(struct page *pageblock_page,
  738. int start_order, int migratetype)
  739. {
  740. int nr_pageblocks = 1 << (start_order - pageblock_order);
  741. while (nr_pageblocks--) {
  742. set_pageblock_migratetype(pageblock_page, migratetype);
  743. pageblock_page += pageblock_nr_pages;
  744. }
  745. }
  746. /* Remove an element from the buddy allocator from the fallback list */
  747. static inline struct page *
  748. __rmqueue_fallback(struct zone *zone, int order, int start_migratetype)
  749. {
  750. struct free_area * area;
  751. int current_order;
  752. struct page *page;
  753. int migratetype, i;
  754. /* Find the largest possible block of pages in the other list */
  755. for (current_order = MAX_ORDER-1; current_order >= order;
  756. --current_order) {
  757. for (i = 0; i < MIGRATE_TYPES - 1; i++) {
  758. migratetype = fallbacks[start_migratetype][i];
  759. /* MIGRATE_RESERVE handled later if necessary */
  760. if (migratetype == MIGRATE_RESERVE)
  761. continue;
  762. area = &(zone->free_area[current_order]);
  763. if (list_empty(&area->free_list[migratetype]))
  764. continue;
  765. page = list_entry(area->free_list[migratetype].next,
  766. struct page, lru);
  767. area->nr_free--;
  768. /*
  769. * If breaking a large block of pages, move all free
  770. * pages to the preferred allocation list. If falling
  771. * back for a reclaimable kernel allocation, be more
  772. * agressive about taking ownership of free pages
  773. */
  774. if (unlikely(current_order >= (pageblock_order >> 1)) ||
  775. start_migratetype == MIGRATE_RECLAIMABLE ||
  776. page_group_by_mobility_disabled) {
  777. unsigned long pages;
  778. pages = move_freepages_block(zone, page,
  779. start_migratetype);
  780. /* Claim the whole block if over half of it is free */
  781. if (pages >= (1 << (pageblock_order-1)) ||
  782. page_group_by_mobility_disabled)
  783. set_pageblock_migratetype(page,
  784. start_migratetype);
  785. migratetype = start_migratetype;
  786. }
  787. /* Remove the page from the freelists */
  788. list_del(&page->lru);
  789. rmv_page_order(page);
  790. /* Take ownership for orders >= pageblock_order */
  791. if (current_order >= pageblock_order)
  792. change_pageblock_range(page, current_order,
  793. start_migratetype);
  794. expand(zone, page, order, current_order, area, migratetype);
  795. trace_mm_page_alloc_extfrag(page, order, current_order,
  796. start_migratetype, migratetype);
  797. return page;
  798. }
  799. }
  800. return NULL;
  801. }
  802. /*
  803. * Do the hard work of removing an element from the buddy allocator.
  804. * Call me with the zone->lock already held.
  805. */
  806. static struct page *__rmqueue(struct zone *zone, unsigned int order,
  807. int migratetype)
  808. {
  809. struct page *page;
  810. retry_reserve:
  811. page = __rmqueue_smallest(zone, order, migratetype);
  812. if (unlikely(!page) && migratetype != MIGRATE_RESERVE) {
  813. page = __rmqueue_fallback(zone, order, migratetype);
  814. /*
  815. * Use MIGRATE_RESERVE rather than fail an allocation. goto
  816. * is used because __rmqueue_smallest is an inline function
  817. * and we want just one call site
  818. */
  819. if (!page) {
  820. migratetype = MIGRATE_RESERVE;
  821. goto retry_reserve;
  822. }
  823. }
  824. trace_mm_page_alloc_zone_locked(page, order, migratetype);
  825. return page;
  826. }
  827. /*
  828. * Obtain a specified number of elements from the buddy allocator, all under
  829. * a single hold of the lock, for efficiency. Add them to the supplied list.
  830. * Returns the number of new pages which were placed at *list.
  831. */
  832. static int rmqueue_bulk(struct zone *zone, unsigned int order,
  833. unsigned long count, struct list_head *list,
  834. int migratetype, int cold)
  835. {
  836. int i;
  837. spin_lock(&zone->lock);
  838. for (i = 0; i < count; ++i) {
  839. struct page *page = __rmqueue(zone, order, migratetype);
  840. if (unlikely(page == NULL))
  841. break;
  842. /*
  843. * Split buddy pages returned by expand() are received here
  844. * in physical page order. The page is added to the callers and
  845. * list and the list head then moves forward. From the callers
  846. * perspective, the linked list is ordered by page number in
  847. * some conditions. This is useful for IO devices that can
  848. * merge IO requests if the physical pages are ordered
  849. * properly.
  850. */
  851. if (likely(cold == 0))
  852. list_add(&page->lru, list);
  853. else
  854. list_add_tail(&page->lru, list);
  855. set_page_private(page, migratetype);
  856. list = &page->lru;
  857. }
  858. __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
  859. spin_unlock(&zone->lock);
  860. return i;
  861. }
  862. #ifdef CONFIG_NUMA
  863. /*
  864. * Called from the vmstat counter updater to drain pagesets of this
  865. * currently executing processor on remote nodes after they have
  866. * expired.
  867. *
  868. * Note that this function must be called with the thread pinned to
  869. * a single processor.
  870. */
  871. void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
  872. {
  873. unsigned long flags;
  874. int to_drain;
  875. local_irq_save(flags);
  876. if (pcp->count >= pcp->batch)
  877. to_drain = pcp->batch;
  878. else
  879. to_drain = pcp->count;
  880. free_pcppages_bulk(zone, to_drain, pcp);
  881. pcp->count -= to_drain;
  882. local_irq_restore(flags);
  883. }
  884. #endif
  885. /*
  886. * Drain pages of the indicated processor.
  887. *
  888. * The processor must either be the current processor and the
  889. * thread pinned to the current processor or a processor that
  890. * is not online.
  891. */
  892. static void drain_pages(unsigned int cpu)
  893. {
  894. unsigned long flags;
  895. struct zone *zone;
  896. for_each_populated_zone(zone) {
  897. struct per_cpu_pageset *pset;
  898. struct per_cpu_pages *pcp;
  899. local_irq_save(flags);
  900. pset = per_cpu_ptr(zone->pageset, cpu);
  901. pcp = &pset->pcp;
  902. free_pcppages_bulk(zone, pcp->count, pcp);
  903. pcp->count = 0;
  904. local_irq_restore(flags);
  905. }
  906. }
  907. /*
  908. * Spill all of this CPU's per-cpu pages back into the buddy allocator.
  909. */
  910. void drain_local_pages(void *arg)
  911. {
  912. drain_pages(smp_processor_id());
  913. }
  914. /*
  915. * Spill all the per-cpu pages from all CPUs back into the buddy allocator
  916. */
  917. void drain_all_pages(void)
  918. {
  919. on_each_cpu(drain_local_pages, NULL, 1);
  920. }
  921. #ifdef CONFIG_HIBERNATION
  922. void mark_free_pages(struct zone *zone)
  923. {
  924. unsigned long pfn, max_zone_pfn;
  925. unsigned long flags;
  926. int order, t;
  927. struct list_head *curr;
  928. if (!zone->spanned_pages)
  929. return;
  930. spin_lock_irqsave(&zone->lock, flags);
  931. max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
  932. for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
  933. if (pfn_valid(pfn)) {
  934. struct page *page = pfn_to_page(pfn);
  935. if (!swsusp_page_is_forbidden(page))
  936. swsusp_unset_page_free(page);
  937. }
  938. for_each_migratetype_order(order, t) {
  939. list_for_each(curr, &zone->free_area[order].free_list[t]) {
  940. unsigned long i;
  941. pfn = page_to_pfn(list_entry(curr, struct page, lru));
  942. for (i = 0; i < (1UL << order); i++)
  943. swsusp_set_page_free(pfn_to_page(pfn + i));
  944. }
  945. }
  946. spin_unlock_irqrestore(&zone->lock, flags);
  947. }
  948. #endif /* CONFIG_PM */
  949. /*
  950. * Free a 0-order page
  951. * cold == 1 ? free a cold page : free a hot page
  952. */
  953. void free_hot_cold_page(struct page *page, int cold)
  954. {
  955. struct zone *zone = page_zone(page);
  956. struct per_cpu_pages *pcp;
  957. unsigned long flags;
  958. int migratetype;
  959. int wasMlocked = __TestClearPageMlocked(page);
  960. trace_mm_page_free_direct(page, 0);
  961. kmemcheck_free_shadow(page, 0);
  962. if (PageAnon(page))
  963. page->mapping = NULL;
  964. if (free_pages_check(page))
  965. return;
  966. if (!PageHighMem(page)) {
  967. debug_check_no_locks_freed(page_address(page), PAGE_SIZE);
  968. debug_check_no_obj_freed(page_address(page), PAGE_SIZE);
  969. }
  970. arch_free_page(page, 0);
  971. kernel_map_pages(page, 1, 0);
  972. migratetype = get_pageblock_migratetype(page);
  973. set_page_private(page, migratetype);
  974. local_irq_save(flags);
  975. if (unlikely(wasMlocked))
  976. free_page_mlock(page);
  977. __count_vm_event(PGFREE);
  978. /*
  979. * We only track unmovable, reclaimable and movable on pcp lists.
  980. * Free ISOLATE pages back to the allocator because they are being
  981. * offlined but treat RESERVE as movable pages so we can get those
  982. * areas back if necessary. Otherwise, we may have to free
  983. * excessively into the page allocator
  984. */
  985. if (migratetype >= MIGRATE_PCPTYPES) {
  986. if (unlikely(migratetype == MIGRATE_ISOLATE)) {
  987. free_one_page(zone, page, 0, migratetype);
  988. goto out;
  989. }
  990. migratetype = MIGRATE_MOVABLE;
  991. }
  992. pcp = &this_cpu_ptr(zone->pageset)->pcp;
  993. if (cold)
  994. list_add_tail(&page->lru, &pcp->lists[migratetype]);
  995. else
  996. list_add(&page->lru, &pcp->lists[migratetype]);
  997. pcp->count++;
  998. if (pcp->count >= pcp->high) {
  999. free_pcppages_bulk(zone, pcp->batch, pcp);
  1000. pcp->count -= pcp->batch;
  1001. }
  1002. out:
  1003. local_irq_restore(flags);
  1004. }
  1005. /*
  1006. * split_page takes a non-compound higher-order page, and splits it into
  1007. * n (1<<order) sub-pages: page[0..n]
  1008. * Each sub-page must be freed individually.
  1009. *
  1010. * Note: this is probably too low level an operation for use in drivers.
  1011. * Please consult with lkml before using this in your driver.
  1012. */
  1013. void split_page(struct page *page, unsigned int order)
  1014. {
  1015. int i;
  1016. VM_BUG_ON(PageCompound(page));
  1017. VM_BUG_ON(!page_count(page));
  1018. #ifdef CONFIG_KMEMCHECK
  1019. /*
  1020. * Split shadow pages too, because free(page[0]) would
  1021. * otherwise free the whole shadow.
  1022. */
  1023. if (kmemcheck_page_is_tracked(page))
  1024. split_page(virt_to_page(page[0].shadow), order);
  1025. #endif
  1026. for (i = 1; i < (1 << order); i++)
  1027. set_page_refcounted(page + i);
  1028. }
  1029. /*
  1030. * Really, prep_compound_page() should be called from __rmqueue_bulk(). But
  1031. * we cheat by calling it from here, in the order > 0 path. Saves a branch
  1032. * or two.
  1033. */
  1034. static inline
  1035. struct page *buffered_rmqueue(struct zone *preferred_zone,
  1036. struct zone *zone, int order, gfp_t gfp_flags,
  1037. int migratetype)
  1038. {
  1039. unsigned long flags;
  1040. struct page *page;
  1041. int cold = !!(gfp_flags & __GFP_COLD);
  1042. again:
  1043. if (likely(order == 0)) {
  1044. struct per_cpu_pages *pcp;
  1045. struct list_head *list;
  1046. local_irq_save(flags);
  1047. pcp = &this_cpu_ptr(zone->pageset)->pcp;
  1048. list = &pcp->lists[migratetype];
  1049. if (list_empty(list)) {
  1050. pcp->count += rmqueue_bulk(zone, 0,
  1051. pcp->batch, list,
  1052. migratetype, cold);
  1053. if (unlikely(list_empty(list)))
  1054. goto failed;
  1055. }
  1056. if (cold)
  1057. page = list_entry(list->prev, struct page, lru);
  1058. else
  1059. page = list_entry(list->next, struct page, lru);
  1060. list_del(&page->lru);
  1061. pcp->count--;
  1062. } else {
  1063. if (unlikely(gfp_flags & __GFP_NOFAIL)) {
  1064. /*
  1065. * __GFP_NOFAIL is not to be used in new code.
  1066. *
  1067. * All __GFP_NOFAIL callers should be fixed so that they
  1068. * properly detect and handle allocation failures.
  1069. *
  1070. * We most definitely don't want callers attempting to
  1071. * allocate greater than order-1 page units with
  1072. * __GFP_NOFAIL.
  1073. */
  1074. WARN_ON_ONCE(order > 1);
  1075. }
  1076. spin_lock_irqsave(&zone->lock, flags);
  1077. page = __rmqueue(zone, order, migratetype);
  1078. spin_unlock(&zone->lock);
  1079. if (!page)
  1080. goto failed;
  1081. __mod_zone_page_state(zone, NR_FREE_PAGES, -(1 << order));
  1082. }
  1083. __count_zone_vm_events(PGALLOC, zone, 1 << order);
  1084. zone_statistics(preferred_zone, zone);
  1085. local_irq_restore(flags);
  1086. VM_BUG_ON(bad_range(zone, page));
  1087. if (prep_new_page(page, order, gfp_flags))
  1088. goto again;
  1089. return page;
  1090. failed:
  1091. local_irq_restore(flags);
  1092. return NULL;
  1093. }
  1094. /* The ALLOC_WMARK bits are used as an index to zone->watermark */
  1095. #define ALLOC_WMARK_MIN WMARK_MIN
  1096. #define ALLOC_WMARK_LOW WMARK_LOW
  1097. #define ALLOC_WMARK_HIGH WMARK_HIGH
  1098. #define ALLOC_NO_WATERMARKS 0x04 /* don't check watermarks at all */
  1099. /* Mask to get the watermark bits */
  1100. #define ALLOC_WMARK_MASK (ALLOC_NO_WATERMARKS-1)
  1101. #define ALLOC_HARDER 0x10 /* try to alloc harder */
  1102. #define ALLOC_HIGH 0x20 /* __GFP_HIGH set */
  1103. #define ALLOC_CPUSET 0x40 /* check for correct cpuset */
  1104. #ifdef CONFIG_FAIL_PAGE_ALLOC
  1105. static struct fail_page_alloc_attr {
  1106. struct fault_attr attr;
  1107. u32 ignore_gfp_highmem;
  1108. u32 ignore_gfp_wait;
  1109. u32 min_order;
  1110. #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
  1111. struct dentry *ignore_gfp_highmem_file;
  1112. struct dentry *ignore_gfp_wait_file;
  1113. struct dentry *min_order_file;
  1114. #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
  1115. } fail_page_alloc = {
  1116. .attr = FAULT_ATTR_INITIALIZER,
  1117. .ignore_gfp_wait = 1,
  1118. .ignore_gfp_highmem = 1,
  1119. .min_order = 1,
  1120. };
  1121. static int __init setup_fail_page_alloc(char *str)
  1122. {
  1123. return setup_fault_attr(&fail_page_alloc.attr, str);
  1124. }
  1125. __setup("fail_page_alloc=", setup_fail_page_alloc);
  1126. static int should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
  1127. {
  1128. if (order < fail_page_alloc.min_order)
  1129. return 0;
  1130. if (gfp_mask & __GFP_NOFAIL)
  1131. return 0;
  1132. if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
  1133. return 0;
  1134. if (fail_page_alloc.ignore_gfp_wait && (gfp_mask & __GFP_WAIT))
  1135. return 0;
  1136. return should_fail(&fail_page_alloc.attr, 1 << order);
  1137. }
  1138. #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
  1139. static int __init fail_page_alloc_debugfs(void)
  1140. {
  1141. mode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
  1142. struct dentry *dir;
  1143. int err;
  1144. err = init_fault_attr_dentries(&fail_page_alloc.attr,
  1145. "fail_page_alloc");
  1146. if (err)
  1147. return err;
  1148. dir = fail_page_alloc.attr.dentries.dir;
  1149. fail_page_alloc.ignore_gfp_wait_file =
  1150. debugfs_create_bool("ignore-gfp-wait", mode, dir,
  1151. &fail_page_alloc.ignore_gfp_wait);
  1152. fail_page_alloc.ignore_gfp_highmem_file =
  1153. debugfs_create_bool("ignore-gfp-highmem", mode, dir,
  1154. &fail_page_alloc.ignore_gfp_highmem);
  1155. fail_page_alloc.min_order_file =
  1156. debugfs_create_u32("min-order", mode, dir,
  1157. &fail_page_alloc.min_order);
  1158. if (!fail_page_alloc.ignore_gfp_wait_file ||
  1159. !fail_page_alloc.ignore_gfp_highmem_file ||
  1160. !fail_page_alloc.min_order_file) {
  1161. err = -ENOMEM;
  1162. debugfs_remove(fail_page_alloc.ignore_gfp_wait_file);
  1163. debugfs_remove(fail_page_alloc.ignore_gfp_highmem_file);
  1164. debugfs_remove(fail_page_alloc.min_order_file);
  1165. cleanup_fault_attr_dentries(&fail_page_alloc.attr);
  1166. }
  1167. return err;
  1168. }
  1169. late_initcall(fail_page_alloc_debugfs);
  1170. #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
  1171. #else /* CONFIG_FAIL_PAGE_ALLOC */
  1172. static inline int should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
  1173. {
  1174. return 0;
  1175. }
  1176. #endif /* CONFIG_FAIL_PAGE_ALLOC */
  1177. /*
  1178. * Return 1 if free pages are above 'mark'. This takes into account the order
  1179. * of the allocation.
  1180. */
  1181. int zone_watermark_ok(struct zone *z, int order, unsigned long mark,
  1182. int classzone_idx, int alloc_flags)
  1183. {
  1184. /* free_pages my go negative - that's OK */
  1185. long min = mark;
  1186. long free_pages = zone_page_state(z, NR_FREE_PAGES) - (1 << order) + 1;
  1187. int o;
  1188. if (alloc_flags & ALLOC_HIGH)
  1189. min -= min / 2;
  1190. if (alloc_flags & ALLOC_HARDER)
  1191. min -= min / 4;
  1192. if (free_pages <= min + z->lowmem_reserve[classzone_idx])
  1193. return 0;
  1194. for (o = 0; o < order; o++) {
  1195. /* At the next order, this order's pages become unavailable */
  1196. free_pages -= z->free_area[o].nr_free << o;
  1197. /* Require fewer higher order pages to be free */
  1198. min >>= 1;
  1199. if (free_pages <= min)
  1200. return 0;
  1201. }
  1202. return 1;
  1203. }
  1204. #ifdef CONFIG_NUMA
  1205. /*
  1206. * zlc_setup - Setup for "zonelist cache". Uses cached zone data to
  1207. * skip over zones that are not allowed by the cpuset, or that have
  1208. * been recently (in last second) found to be nearly full. See further
  1209. * comments in mmzone.h. Reduces cache footprint of zonelist scans
  1210. * that have to skip over a lot of full or unallowed zones.
  1211. *
  1212. * If the zonelist cache is present in the passed in zonelist, then
  1213. * returns a pointer to the allowed node mask (either the current
  1214. * tasks mems_allowed, or node_states[N_HIGH_MEMORY].)
  1215. *
  1216. * If the zonelist cache is not available for this zonelist, does
  1217. * nothing and returns NULL.
  1218. *
  1219. * If the fullzones BITMAP in the zonelist cache is stale (more than
  1220. * a second since last zap'd) then we zap it out (clear its bits.)
  1221. *
  1222. * We hold off even calling zlc_setup, until after we've checked the
  1223. * first zone in the zonelist, on the theory that most allocations will
  1224. * be satisfied from that first zone, so best to examine that zone as
  1225. * quickly as we can.
  1226. */
  1227. static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
  1228. {
  1229. struct zonelist_cache *zlc; /* cached zonelist speedup info */
  1230. nodemask_t *allowednodes; /* zonelist_cache approximation */
  1231. zlc = zonelist->zlcache_ptr;
  1232. if (!zlc)
  1233. return NULL;
  1234. if (time_after(jiffies, zlc->last_full_zap + HZ)) {
  1235. bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
  1236. zlc->last_full_zap = jiffies;
  1237. }
  1238. allowednodes = !in_interrupt() && (alloc_flags & ALLOC_CPUSET) ?
  1239. &cpuset_current_mems_allowed :
  1240. &node_states[N_HIGH_MEMORY];
  1241. return allowednodes;
  1242. }
  1243. /*
  1244. * Given 'z' scanning a zonelist, run a couple of quick checks to see
  1245. * if it is worth looking at further for free memory:
  1246. * 1) Check that the zone isn't thought to be full (doesn't have its
  1247. * bit set in the zonelist_cache fullzones BITMAP).
  1248. * 2) Check that the zones node (obtained from the zonelist_cache
  1249. * z_to_n[] mapping) is allowed in the passed in allowednodes mask.
  1250. * Return true (non-zero) if zone is worth looking at further, or
  1251. * else return false (zero) if it is not.
  1252. *
  1253. * This check -ignores- the distinction between various watermarks,
  1254. * such as GFP_HIGH, GFP_ATOMIC, PF_MEMALLOC, ... If a zone is
  1255. * found to be full for any variation of these watermarks, it will
  1256. * be considered full for up to one second by all requests, unless
  1257. * we are so low on memory on all allowed nodes that we are forced
  1258. * into the second scan of the zonelist.
  1259. *
  1260. * In the second scan we ignore this zonelist cache and exactly
  1261. * apply the watermarks to all zones, even it is slower to do so.
  1262. * We are low on memory in the second scan, and should leave no stone
  1263. * unturned looking for a free page.
  1264. */
  1265. static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z,
  1266. nodemask_t *allowednodes)
  1267. {
  1268. struct zonelist_cache *zlc; /* cached zonelist speedup info */
  1269. int i; /* index of *z in zonelist zones */
  1270. int n; /* node that zone *z is on */
  1271. zlc = zonelist->zlcache_ptr;
  1272. if (!zlc)
  1273. return 1;
  1274. i = z - zonelist->_zonerefs;
  1275. n = zlc->z_to_n[i];
  1276. /* This zone is worth trying if it is allowed but not full */
  1277. return node_isset(n, *allowednodes) && !test_bit(i, zlc->fullzones);
  1278. }
  1279. /*
  1280. * Given 'z' scanning a zonelist, set the corresponding bit in
  1281. * zlc->fullzones, so that subsequent attempts to allocate a page
  1282. * from that zone don't waste time re-examining it.
  1283. */
  1284. static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z)
  1285. {
  1286. struct zonelist_cache *zlc; /* cached zonelist speedup info */
  1287. int i; /* index of *z in zonelist zones */
  1288. zlc = zonelist->zlcache_ptr;
  1289. if (!zlc)
  1290. return;
  1291. i = z - zonelist->_zonerefs;
  1292. set_bit(i, zlc->fullzones);
  1293. }
  1294. #else /* CONFIG_NUMA */
  1295. static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
  1296. {
  1297. return NULL;
  1298. }
  1299. static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z,
  1300. nodemask_t *allowednodes)
  1301. {
  1302. return 1;
  1303. }
  1304. static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z)
  1305. {
  1306. }
  1307. #endif /* CONFIG_NUMA */
  1308. /*
  1309. * get_page_from_freelist goes through the zonelist trying to allocate
  1310. * a page.
  1311. */
  1312. static struct page *
  1313. get_page_from_freelist(gfp_t gfp_mask, nodemask_t *nodemask, unsigned int order,
  1314. struct zonelist *zonelist, int high_zoneidx, int alloc_flags,
  1315. struct zone *preferred_zone, int migratetype)
  1316. {
  1317. struct zoneref *z;
  1318. struct page *page = NULL;
  1319. int classzone_idx;
  1320. struct zone *zone;
  1321. nodemask_t *allowednodes = NULL;/* zonelist_cache approximation */
  1322. int zlc_active = 0; /* set if using zonelist_cache */
  1323. int did_zlc_setup = 0; /* just call zlc_setup() one time */
  1324. classzone_idx = zone_idx(preferred_zone);
  1325. zonelist_scan:
  1326. /*
  1327. * Scan zonelist, looking for a zone with enough free.
  1328. * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
  1329. */
  1330. for_each_zone_zonelist_nodemask(zone, z, zonelist,
  1331. high_zoneidx, nodemask) {
  1332. if (NUMA_BUILD && zlc_active &&
  1333. !zlc_zone_worth_trying(zonelist, z, allowednodes))
  1334. continue;
  1335. if ((alloc_flags & ALLOC_CPUSET) &&
  1336. !cpuset_zone_allowed_softwall(zone, gfp_mask))
  1337. goto try_next_zone;
  1338. BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
  1339. if (!(alloc_flags & ALLOC_NO_WATERMARKS)) {
  1340. unsigned long mark;
  1341. int ret;
  1342. mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK];
  1343. if (zone_watermark_ok(zone, order, mark,
  1344. classzone_idx, alloc_flags))
  1345. goto try_this_zone;
  1346. if (zone_reclaim_mode == 0)
  1347. goto this_zone_full;
  1348. ret = zone_reclaim(zone, gfp_mask, order);
  1349. switch (ret) {
  1350. case ZONE_RECLAIM_NOSCAN:
  1351. /* did not scan */
  1352. goto try_next_zone;
  1353. case ZONE_RECLAIM_FULL:
  1354. /* scanned but unreclaimable */
  1355. goto this_zone_full;
  1356. default:
  1357. /* did we reclaim enough */
  1358. if (!zone_watermark_ok(zone, order, mark,
  1359. classzone_idx, alloc_flags))
  1360. goto this_zone_full;
  1361. }
  1362. }
  1363. try_this_zone:
  1364. page = buffered_rmqueue(preferred_zone, zone, order,
  1365. gfp_mask, migratetype);
  1366. if (page)
  1367. break;
  1368. this_zone_full:
  1369. if (NUMA_BUILD)
  1370. zlc_mark_zone_full(zonelist, z);
  1371. try_next_zone:
  1372. if (NUMA_BUILD && !did_zlc_setup && nr_online_nodes > 1) {
  1373. /*
  1374. * we do zlc_setup after the first zone is tried but only
  1375. * if there are multiple nodes make it worthwhile
  1376. */
  1377. allowednodes = zlc_setup(zonelist, alloc_flags);
  1378. zlc_active = 1;
  1379. did_zlc_setup = 1;
  1380. }
  1381. }
  1382. if (unlikely(NUMA_BUILD && page == NULL && zlc_active)) {
  1383. /* Disable zlc cache for second zonelist scan */
  1384. zlc_active = 0;
  1385. goto zonelist_scan;
  1386. }
  1387. return page;
  1388. }
  1389. static inline int
  1390. should_alloc_retry(gfp_t gfp_mask, unsigned int order,
  1391. unsigned long pages_reclaimed)
  1392. {
  1393. /* Do not loop if specifically requested */
  1394. if (gfp_mask & __GFP_NORETRY)
  1395. return 0;
  1396. /*
  1397. * In this implementation, order <= PAGE_ALLOC_COSTLY_ORDER
  1398. * means __GFP_NOFAIL, but that may not be true in other
  1399. * implementations.
  1400. */
  1401. if (order <= PAGE_ALLOC_COSTLY_ORDER)
  1402. return 1;
  1403. /*
  1404. * For order > PAGE_ALLOC_COSTLY_ORDER, if __GFP_REPEAT is
  1405. * specified, then we retry until we no longer reclaim any pages
  1406. * (above), or we've reclaimed an order of pages at least as
  1407. * large as the allocation's order. In both cases, if the
  1408. * allocation still fails, we stop retrying.
  1409. */
  1410. if (gfp_mask & __GFP_REPEAT && pages_reclaimed < (1 << order))
  1411. return 1;
  1412. /*
  1413. * Don't let big-order allocations loop unless the caller
  1414. * explicitly requests that.
  1415. */
  1416. if (gfp_mask & __GFP_NOFAIL)
  1417. return 1;
  1418. return 0;
  1419. }
  1420. static inline struct page *
  1421. __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
  1422. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1423. nodemask_t *nodemask, struct zone *preferred_zone,
  1424. int migratetype)
  1425. {
  1426. struct page *page;
  1427. /* Acquire the OOM killer lock for the zones in zonelist */
  1428. if (!try_set_zone_oom(zonelist, gfp_mask)) {
  1429. schedule_timeout_uninterruptible(1);
  1430. return NULL;
  1431. }
  1432. /*
  1433. * Go through the zonelist yet one more time, keep very high watermark
  1434. * here, this is only to catch a parallel oom killing, we must fail if
  1435. * we're still under heavy pressure.
  1436. */
  1437. page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask,
  1438. order, zonelist, high_zoneidx,
  1439. ALLOC_WMARK_HIGH|ALLOC_CPUSET,
  1440. preferred_zone, migratetype);
  1441. if (page)
  1442. goto out;
  1443. if (!(gfp_mask & __GFP_NOFAIL)) {
  1444. /* The OOM killer will not help higher order allocs */
  1445. if (order > PAGE_ALLOC_COSTLY_ORDER)
  1446. goto out;
  1447. /*
  1448. * GFP_THISNODE contains __GFP_NORETRY and we never hit this.
  1449. * Sanity check for bare calls of __GFP_THISNODE, not real OOM.
  1450. * The caller should handle page allocation failure by itself if
  1451. * it specifies __GFP_THISNODE.
  1452. * Note: Hugepage uses it but will hit PAGE_ALLOC_COSTLY_ORDER.
  1453. */
  1454. if (gfp_mask & __GFP_THISNODE)
  1455. goto out;
  1456. }
  1457. /* Exhausted what can be done so it's blamo time */
  1458. out_of_memory(zonelist, gfp_mask, order, nodemask);
  1459. out:
  1460. clear_zonelist_oom(zonelist, gfp_mask);
  1461. return page;
  1462. }
  1463. /* The really slow allocator path where we enter direct reclaim */
  1464. static inline struct page *
  1465. __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
  1466. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1467. nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
  1468. int migratetype, unsigned long *did_some_progress)
  1469. {
  1470. struct page *page = NULL;
  1471. struct reclaim_state reclaim_state;
  1472. struct task_struct *p = current;
  1473. cond_resched();
  1474. /* We now go into synchronous reclaim */
  1475. cpuset_memory_pressure_bump();
  1476. p->flags |= PF_MEMALLOC;
  1477. lockdep_set_current_reclaim_state(gfp_mask);
  1478. reclaim_state.reclaimed_slab = 0;
  1479. p->reclaim_state = &reclaim_state;
  1480. *did_some_progress = try_to_free_pages(zonelist, order, gfp_mask, nodemask);
  1481. p->reclaim_state = NULL;
  1482. lockdep_clear_current_reclaim_state();
  1483. p->flags &= ~PF_MEMALLOC;
  1484. cond_resched();
  1485. if (order != 0)
  1486. drain_all_pages();
  1487. if (likely(*did_some_progress))
  1488. page = get_page_from_freelist(gfp_mask, nodemask, order,
  1489. zonelist, high_zoneidx,
  1490. alloc_flags, preferred_zone,
  1491. migratetype);
  1492. return page;
  1493. }
  1494. /*
  1495. * This is called in the allocator slow-path if the allocation request is of
  1496. * sufficient urgency to ignore watermarks and take other desperate measures
  1497. */
  1498. static inline struct page *
  1499. __alloc_pages_high_priority(gfp_t gfp_mask, unsigned int order,
  1500. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1501. nodemask_t *nodemask, struct zone *preferred_zone,
  1502. int migratetype)
  1503. {
  1504. struct page *page;
  1505. do {
  1506. page = get_page_from_freelist(gfp_mask, nodemask, order,
  1507. zonelist, high_zoneidx, ALLOC_NO_WATERMARKS,
  1508. preferred_zone, migratetype);
  1509. if (!page && gfp_mask & __GFP_NOFAIL)
  1510. congestion_wait(BLK_RW_ASYNC, HZ/50);
  1511. } while (!page && (gfp_mask & __GFP_NOFAIL));
  1512. return page;
  1513. }
  1514. static inline
  1515. void wake_all_kswapd(unsigned int order, struct zonelist *zonelist,
  1516. enum zone_type high_zoneidx)
  1517. {
  1518. struct zoneref *z;
  1519. struct zone *zone;
  1520. for_each_zone_zonelist(zone, z, zonelist, high_zoneidx)
  1521. wakeup_kswapd(zone, order);
  1522. }
  1523. static inline int
  1524. gfp_to_alloc_flags(gfp_t gfp_mask)
  1525. {
  1526. struct task_struct *p = current;
  1527. int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
  1528. const gfp_t wait = gfp_mask & __GFP_WAIT;
  1529. /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */
  1530. BUILD_BUG_ON(__GFP_HIGH != ALLOC_HIGH);
  1531. /*
  1532. * The caller may dip into page reserves a bit more if the caller
  1533. * cannot run direct reclaim, or if the caller has realtime scheduling
  1534. * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will
  1535. * set both ALLOC_HARDER (!wait) and ALLOC_HIGH (__GFP_HIGH).
  1536. */
  1537. alloc_flags |= (gfp_mask & __GFP_HIGH);
  1538. if (!wait) {
  1539. alloc_flags |= ALLOC_HARDER;
  1540. /*
  1541. * Ignore cpuset if GFP_ATOMIC (!wait) rather than fail alloc.
  1542. * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
  1543. */
  1544. alloc_flags &= ~ALLOC_CPUSET;
  1545. } else if (unlikely(rt_task(p)) && !in_interrupt())
  1546. alloc_flags |= ALLOC_HARDER;
  1547. if (likely(!(gfp_mask & __GFP_NOMEMALLOC))) {
  1548. if (!in_interrupt() &&
  1549. ((p->flags & PF_MEMALLOC) ||
  1550. unlikely(test_thread_flag(TIF_MEMDIE))))
  1551. alloc_flags |= ALLOC_NO_WATERMARKS;
  1552. }
  1553. return alloc_flags;
  1554. }
  1555. static inline struct page *
  1556. __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
  1557. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1558. nodemask_t *nodemask, struct zone *preferred_zone,
  1559. int migratetype)
  1560. {
  1561. const gfp_t wait = gfp_mask & __GFP_WAIT;
  1562. struct page *page = NULL;
  1563. int alloc_flags;
  1564. unsigned long pages_reclaimed = 0;
  1565. unsigned long did_some_progress;
  1566. struct task_struct *p = current;
  1567. /*
  1568. * In the slowpath, we sanity check order to avoid ever trying to
  1569. * reclaim >= MAX_ORDER areas which will never succeed. Callers may
  1570. * be using allocators in order of preference for an area that is
  1571. * too large.
  1572. */
  1573. if (order >= MAX_ORDER) {
  1574. WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));
  1575. return NULL;
  1576. }
  1577. /*
  1578. * GFP_THISNODE (meaning __GFP_THISNODE, __GFP_NORETRY and
  1579. * __GFP_NOWARN set) should not cause reclaim since the subsystem
  1580. * (f.e. slab) using GFP_THISNODE may choose to trigger reclaim
  1581. * using a larger set of nodes after it has established that the
  1582. * allowed per node queues are empty and that nodes are
  1583. * over allocated.
  1584. */
  1585. if (NUMA_BUILD && (gfp_mask & GFP_THISNODE) == GFP_THISNODE)
  1586. goto nopage;
  1587. restart:
  1588. wake_all_kswapd(order, zonelist, high_zoneidx);
  1589. /*
  1590. * OK, we're below the kswapd watermark and have kicked background
  1591. * reclaim. Now things get more complex, so set up alloc_flags according
  1592. * to how we want to proceed.
  1593. */
  1594. alloc_flags = gfp_to_alloc_flags(gfp_mask);
  1595. /* This is the last chance, in general, before the goto nopage. */
  1596. page = get_page_from_freelist(gfp_mask, nodemask, order, zonelist,
  1597. high_zoneidx, alloc_flags & ~ALLOC_NO_WATERMARKS,
  1598. preferred_zone, migratetype);
  1599. if (page)
  1600. goto got_pg;
  1601. rebalance:
  1602. /* Allocate without watermarks if the context allows */
  1603. if (alloc_flags & ALLOC_NO_WATERMARKS) {
  1604. page = __alloc_pages_high_priority(gfp_mask, order,
  1605. zonelist, high_zoneidx, nodemask,
  1606. preferred_zone, migratetype);
  1607. if (page)
  1608. goto got_pg;
  1609. }
  1610. /* Atomic allocations - we can't balance anything */
  1611. if (!wait)
  1612. goto nopage;
  1613. /* Avoid recursion of direct reclaim */
  1614. if (p->flags & PF_MEMALLOC)
  1615. goto nopage;
  1616. /* Avoid allocations with no watermarks from looping endlessly */
  1617. if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL))
  1618. goto nopage;
  1619. /* Try direct reclaim and then allocating */
  1620. page = __alloc_pages_direct_reclaim(gfp_mask, order,
  1621. zonelist, high_zoneidx,
  1622. nodemask,
  1623. alloc_flags, preferred_zone,
  1624. migratetype, &did_some_progress);
  1625. if (page)
  1626. goto got_pg;
  1627. /*
  1628. * If we failed to make any progress reclaiming, then we are
  1629. * running out of options and have to consider going OOM
  1630. */
  1631. if (!did_some_progress) {
  1632. if ((gfp_mask & __GFP_FS) && !(gfp_mask & __GFP_NORETRY)) {
  1633. if (oom_killer_disabled)
  1634. goto nopage;
  1635. page = __alloc_pages_may_oom(gfp_mask, order,
  1636. zonelist, high_zoneidx,
  1637. nodemask, preferred_zone,
  1638. migratetype);
  1639. if (page)
  1640. goto got_pg;
  1641. /*
  1642. * The OOM killer does not trigger for high-order
  1643. * ~__GFP_NOFAIL allocations so if no progress is being
  1644. * made, there are no other options and retrying is
  1645. * unlikely to help.
  1646. */
  1647. if (order > PAGE_ALLOC_COSTLY_ORDER &&
  1648. !(gfp_mask & __GFP_NOFAIL))
  1649. goto nopage;
  1650. goto restart;
  1651. }
  1652. }
  1653. /* Check if we should retry the allocation */
  1654. pages_reclaimed += did_some_progress;
  1655. if (should_alloc_retry(gfp_mask, order, pages_reclaimed)) {
  1656. /* Wait for some write requests to complete then retry */
  1657. congestion_wait(BLK_RW_ASYNC, HZ/50);
  1658. goto rebalance;
  1659. }
  1660. nopage:
  1661. if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit()) {
  1662. printk(KERN_WARNING "%s: page allocation failure."
  1663. " order:%d, mode:0x%x\n",
  1664. p->comm, order, gfp_mask);
  1665. dump_stack();
  1666. show_mem();
  1667. }
  1668. return page;
  1669. got_pg:
  1670. if (kmemcheck_enabled)
  1671. kmemcheck_pagealloc_alloc(page, order, gfp_mask);
  1672. return page;
  1673. }
  1674. /*
  1675. * This is the 'heart' of the zoned buddy allocator.
  1676. */
  1677. struct page *
  1678. __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
  1679. struct zonelist *zonelist, nodemask_t *nodemask)
  1680. {
  1681. enum zone_type high_zoneidx = gfp_zone(gfp_mask);
  1682. struct zone *preferred_zone;
  1683. struct page *page;
  1684. int migratetype = allocflags_to_migratetype(gfp_mask);
  1685. gfp_mask &= gfp_allowed_mask;
  1686. lockdep_trace_alloc(gfp_mask);
  1687. might_sleep_if(gfp_mask & __GFP_WAIT);
  1688. if (should_fail_alloc_page(gfp_mask, order))
  1689. return NULL;
  1690. /*
  1691. * Check the zones suitable for the gfp_mask contain at least one
  1692. * valid zone. It's possible to have an empty zonelist as a result
  1693. * of GFP_THISNODE and a memoryless node
  1694. */
  1695. if (unlikely(!zonelist->_zonerefs->zone))
  1696. return NULL;
  1697. /* The preferred zone is used for statistics later */
  1698. first_zones_zonelist(zonelist, high_zoneidx, nodemask, &preferred_zone);
  1699. if (!preferred_zone)
  1700. return NULL;
  1701. /* First allocation attempt */
  1702. page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask, order,
  1703. zonelist, high_zoneidx, ALLOC_WMARK_LOW|ALLOC_CPUSET,
  1704. preferred_zone, migratetype);
  1705. if (unlikely(!page))
  1706. page = __alloc_pages_slowpath(gfp_mask, order,
  1707. zonelist, high_zoneidx, nodemask,
  1708. preferred_zone, migratetype);
  1709. trace_mm_page_alloc(page, order, gfp_mask, migratetype);
  1710. return page;
  1711. }
  1712. EXPORT_SYMBOL(__alloc_pages_nodemask);
  1713. /*
  1714. * Common helper functions.
  1715. */
  1716. unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
  1717. {
  1718. struct page *page;
  1719. /*
  1720. * __get_free_pages() returns a 32-bit address, which cannot represent
  1721. * a highmem page
  1722. */
  1723. VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0);
  1724. page = alloc_pages(gfp_mask, order);
  1725. if (!page)
  1726. return 0;
  1727. return (unsigned long) page_address(page);
  1728. }
  1729. EXPORT_SYMBOL(__get_free_pages);
  1730. unsigned long get_zeroed_page(gfp_t gfp_mask)
  1731. {
  1732. return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
  1733. }
  1734. EXPORT_SYMBOL(get_zeroed_page);
  1735. void __pagevec_free(struct pagevec *pvec)
  1736. {
  1737. int i = pagevec_count(pvec);
  1738. while (--i >= 0) {
  1739. trace_mm_pagevec_free(pvec->pages[i], pvec->cold);
  1740. free_hot_cold_page(pvec->pages[i], pvec->cold);
  1741. }
  1742. }
  1743. void __free_pages(struct page *page, unsigned int order)
  1744. {
  1745. if (put_page_testzero(page)) {
  1746. if (order == 0)
  1747. free_hot_cold_page(page, 0);
  1748. else
  1749. __free_pages_ok(page, order);
  1750. }
  1751. }
  1752. EXPORT_SYMBOL(__free_pages);
  1753. void free_pages(unsigned long addr, unsigned int order)
  1754. {
  1755. if (addr != 0) {
  1756. VM_BUG_ON(!virt_addr_valid((void *)addr));
  1757. __free_pages(virt_to_page((void *)addr), order);
  1758. }
  1759. }
  1760. EXPORT_SYMBOL(free_pages);
  1761. /**
  1762. * alloc_pages_exact - allocate an exact number physically-contiguous pages.
  1763. * @size: the number of bytes to allocate
  1764. * @gfp_mask: GFP flags for the allocation
  1765. *
  1766. * This function is similar to alloc_pages(), except that it allocates the
  1767. * minimum number of pages to satisfy the request. alloc_pages() can only
  1768. * allocate memory in power-of-two pages.
  1769. *
  1770. * This function is also limited by MAX_ORDER.
  1771. *
  1772. * Memory allocated by this function must be released by free_pages_exact().
  1773. */
  1774. void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
  1775. {
  1776. unsigned int order = get_order(size);
  1777. unsigned long addr;
  1778. addr = __get_free_pages(gfp_mask, order);
  1779. if (addr) {
  1780. unsigned long alloc_end = addr + (PAGE_SIZE << order);
  1781. unsigned long used = addr + PAGE_ALIGN(size);
  1782. split_page(virt_to_page((void *)addr), order);
  1783. while (used < alloc_end) {
  1784. free_page(used);
  1785. used += PAGE_SIZE;
  1786. }
  1787. }
  1788. return (void *)addr;
  1789. }
  1790. EXPORT_SYMBOL(alloc_pages_exact);
  1791. /**
  1792. * free_pages_exact - release memory allocated via alloc_pages_exact()
  1793. * @virt: the value returned by alloc_pages_exact.
  1794. * @size: size of allocation, same value as passed to alloc_pages_exact().
  1795. *
  1796. * Release the memory allocated by a previous call to alloc_pages_exact.
  1797. */
  1798. void free_pages_exact(void *virt, size_t size)
  1799. {
  1800. unsigned long addr = (unsigned long)virt;
  1801. unsigned long end = addr + PAGE_ALIGN(size);
  1802. while (addr < end) {
  1803. free_page(addr);
  1804. addr += PAGE_SIZE;
  1805. }
  1806. }
  1807. EXPORT_SYMBOL(free_pages_exact);
  1808. static unsigned int nr_free_zone_pages(int offset)
  1809. {
  1810. struct zoneref *z;
  1811. struct zone *zone;
  1812. /* Just pick one node, since fallback list is circular */
  1813. unsigned int sum = 0;
  1814. struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
  1815. for_each_zone_zonelist(zone, z, zonelist, offset) {
  1816. unsigned long size = zone->present_pages;
  1817. unsigned long high = high_wmark_pages(zone);
  1818. if (size > high)
  1819. sum += size - high;
  1820. }
  1821. return sum;
  1822. }
  1823. /*
  1824. * Amount of free RAM allocatable within ZONE_DMA and ZONE_NORMAL
  1825. */
  1826. unsigned int nr_free_buffer_pages(void)
  1827. {
  1828. return nr_free_zone_pages(gfp_zone(GFP_USER));
  1829. }
  1830. EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
  1831. /*
  1832. * Amount of free RAM allocatable within all zones
  1833. */
  1834. unsigned int nr_free_pagecache_pages(void)
  1835. {
  1836. return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
  1837. }
  1838. static inline void show_node(struct zone *zone)
  1839. {
  1840. if (NUMA_BUILD)
  1841. printk("Node %d ", zone_to_nid(zone));
  1842. }
  1843. void si_meminfo(struct sysinfo *val)
  1844. {
  1845. val->totalram = totalram_pages;
  1846. val->sharedram = 0;
  1847. val->freeram = global_page_state(NR_FREE_PAGES);
  1848. val->bufferram = nr_blockdev_pages();
  1849. val->totalhigh = totalhigh_pages;
  1850. val->freehigh = nr_free_highpages();
  1851. val->mem_unit = PAGE_SIZE;
  1852. }
  1853. EXPORT_SYMBOL(si_meminfo);
  1854. #ifdef CONFIG_NUMA
  1855. void si_meminfo_node(struct sysinfo *val, int nid)
  1856. {
  1857. pg_data_t *pgdat = NODE_DATA(nid);
  1858. val->totalram = pgdat->node_present_pages;
  1859. val->freeram = node_page_state(nid, NR_FREE_PAGES);
  1860. #ifdef CONFIG_HIGHMEM
  1861. val->totalhigh = pgdat->node_zones[ZONE_HIGHMEM].present_pages;
  1862. val->freehigh = zone_page_state(&pgdat->node_zones[ZONE_HIGHMEM],
  1863. NR_FREE_PAGES);
  1864. #else
  1865. val->totalhigh = 0;
  1866. val->freehigh = 0;
  1867. #endif
  1868. val->mem_unit = PAGE_SIZE;
  1869. }
  1870. #endif
  1871. #define K(x) ((x) << (PAGE_SHIFT-10))
  1872. /*
  1873. * Show free area list (used inside shift_scroll-lock stuff)
  1874. * We also calculate the percentage fragmentation. We do this by counting the
  1875. * memory on each free list with the exception of the first item on the list.
  1876. */
  1877. void show_free_areas(void)
  1878. {
  1879. int cpu;
  1880. struct zone *zone;
  1881. for_each_populated_zone(zone) {
  1882. show_node(zone);
  1883. printk("%s per-cpu:\n", zone->name);
  1884. for_each_online_cpu(cpu) {
  1885. struct per_cpu_pageset *pageset;
  1886. pageset = per_cpu_ptr(zone->pageset, cpu);
  1887. printk("CPU %4d: hi:%5d, btch:%4d usd:%4d\n",
  1888. cpu, pageset->pcp.high,
  1889. pageset->pcp.batch, pageset->pcp.count);
  1890. }
  1891. }
  1892. printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n"
  1893. " active_file:%lu inactive_file:%lu isolated_file:%lu\n"
  1894. " unevictable:%lu"
  1895. " dirty:%lu writeback:%lu unstable:%lu\n"
  1896. " free:%lu slab_reclaimable:%lu slab_unreclaimable:%lu\n"
  1897. " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n",
  1898. global_page_state(NR_ACTIVE_ANON),
  1899. global_page_state(NR_INACTIVE_ANON),
  1900. global_page_state(NR_ISOLATED_ANON),
  1901. global_page_state(NR_ACTIVE_FILE),
  1902. global_page_state(NR_INACTIVE_FILE),
  1903. global_page_state(NR_ISOLATED_FILE),
  1904. global_page_state(NR_UNEVICTABLE),
  1905. global_page_state(NR_FILE_DIRTY),
  1906. global_page_state(NR_WRITEBACK),
  1907. global_page_state(NR_UNSTABLE_NFS),
  1908. global_page_state(NR_FREE_PAGES),
  1909. global_page_state(NR_SLAB_RECLAIMABLE),
  1910. global_page_state(NR_SLAB_UNRECLAIMABLE),
  1911. global_page_state(NR_FILE_MAPPED),
  1912. global_page_state(NR_SHMEM),
  1913. global_page_state(NR_PAGETABLE),
  1914. global_page_state(NR_BOUNCE));
  1915. for_each_populated_zone(zone) {
  1916. int i;
  1917. show_node(zone);
  1918. printk("%s"
  1919. " free:%lukB"
  1920. " min:%lukB"
  1921. " low:%lukB"
  1922. " high:%lukB"
  1923. " active_anon:%lukB"
  1924. " inactive_anon:%lukB"
  1925. " active_file:%lukB"
  1926. " inactive_file:%lukB"
  1927. " unevictable:%lukB"
  1928. " isolated(anon):%lukB"
  1929. " isolated(file):%lukB"
  1930. " present:%lukB"
  1931. " mlocked:%lukB"
  1932. " dirty:%lukB"
  1933. " writeback:%lukB"
  1934. " mapped:%lukB"
  1935. " shmem:%lukB"
  1936. " slab_reclaimable:%lukB"
  1937. " slab_unreclaimable:%lukB"
  1938. " kernel_stack:%lukB"
  1939. " pagetables:%lukB"
  1940. " unstable:%lukB"
  1941. " bounce:%lukB"
  1942. " writeback_tmp:%lukB"
  1943. " pages_scanned:%lu"
  1944. " all_unreclaimable? %s"
  1945. "\n",
  1946. zone->name,
  1947. K(zone_page_state(zone, NR_FREE_PAGES)),
  1948. K(min_wmark_pages(zone)),
  1949. K(low_wmark_pages(zone)),
  1950. K(high_wmark_pages(zone)),
  1951. K(zone_page_state(zone, NR_ACTIVE_ANON)),
  1952. K(zone_page_state(zone, NR_INACTIVE_ANON)),
  1953. K(zone_page_state(zone, NR_ACTIVE_FILE)),
  1954. K(zone_page_state(zone, NR_INACTIVE_FILE)),
  1955. K(zone_page_state(zone, NR_UNEVICTABLE)),
  1956. K(zone_page_state(zone, NR_ISOLATED_ANON)),
  1957. K(zone_page_state(zone, NR_ISOLATED_FILE)),
  1958. K(zone->present_pages),
  1959. K(zone_page_state(zone, NR_MLOCK)),
  1960. K(zone_page_state(zone, NR_FILE_DIRTY)),
  1961. K(zone_page_state(zone, NR_WRITEBACK)),
  1962. K(zone_page_state(zone, NR_FILE_MAPPED)),
  1963. K(zone_page_state(zone, NR_SHMEM)),
  1964. K(zone_page_state(zone, NR_SLAB_RECLAIMABLE)),
  1965. K(zone_page_state(zone, NR_SLAB_UNRECLAIMABLE)),
  1966. zone_page_state(zone, NR_KERNEL_STACK) *
  1967. THREAD_SIZE / 1024,
  1968. K(zone_page_state(zone, NR_PAGETABLE)),
  1969. K(zone_page_state(zone, NR_UNSTABLE_NFS)),
  1970. K(zone_page_state(zone, NR_BOUNCE)),
  1971. K(zone_page_state(zone, NR_WRITEBACK_TEMP)),
  1972. zone->pages_scanned,
  1973. (zone->all_unreclaimable ? "yes" : "no")
  1974. );
  1975. printk("lowmem_reserve[]:");
  1976. for (i = 0; i < MAX_NR_ZONES; i++)
  1977. printk(" %lu", zone->lowmem_reserve[i]);
  1978. printk("\n");
  1979. }
  1980. for_each_populated_zone(zone) {
  1981. unsigned long nr[MAX_ORDER], flags, order, total = 0;
  1982. show_node(zone);
  1983. printk("%s: ", zone->name);
  1984. spin_lock_irqsave(&zone->lock, flags);
  1985. for (order = 0; order < MAX_ORDER; order++) {
  1986. nr[order] = zone->free_area[order].nr_free;
  1987. total += nr[order] << order;
  1988. }
  1989. spin_unlock_irqrestore(&zone->lock, flags);
  1990. for (order = 0; order < MAX_ORDER; order++)
  1991. printk("%lu*%lukB ", nr[order], K(1UL) << order);
  1992. printk("= %lukB\n", K(total));
  1993. }
  1994. printk("%ld total pagecache pages\n", global_page_state(NR_FILE_PAGES));
  1995. show_swap_cache_info();
  1996. }
  1997. static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
  1998. {
  1999. zoneref->zone = zone;
  2000. zoneref->zone_idx = zone_idx(zone);
  2001. }
  2002. /*
  2003. * Builds allocation fallback zone lists.
  2004. *
  2005. * Add all populated zones of a node to the zonelist.
  2006. */
  2007. static int build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist,
  2008. int nr_zones, enum zone_type zone_type)
  2009. {
  2010. struct zone *zone;
  2011. BUG_ON(zone_type >= MAX_NR_ZONES);
  2012. zone_type++;
  2013. do {
  2014. zone_type--;
  2015. zone = pgdat->node_zones + zone_type;
  2016. if (populated_zone(zone)) {
  2017. zoneref_set_zone(zone,
  2018. &zonelist->_zonerefs[nr_zones++]);
  2019. check_highest_zone(zone_type);
  2020. }
  2021. } while (zone_type);
  2022. return nr_zones;
  2023. }
  2024. /*
  2025. * zonelist_order:
  2026. * 0 = automatic detection of better ordering.
  2027. * 1 = order by ([node] distance, -zonetype)
  2028. * 2 = order by (-zonetype, [node] distance)
  2029. *
  2030. * If not NUMA, ZONELIST_ORDER_ZONE and ZONELIST_ORDER_NODE will create
  2031. * the same zonelist. So only NUMA can configure this param.
  2032. */
  2033. #define ZONELIST_ORDER_DEFAULT 0
  2034. #define ZONELIST_ORDER_NODE 1
  2035. #define ZONELIST_ORDER_ZONE 2
  2036. /* zonelist order in the kernel.
  2037. * set_zonelist_order() will set this to NODE or ZONE.
  2038. */
  2039. static int current_zonelist_order = ZONELIST_ORDER_DEFAULT;
  2040. static char zonelist_order_name[3][8] = {"Default", "Node", "Zone"};
  2041. #ifdef CONFIG_NUMA
  2042. /* The value user specified ....changed by config */
  2043. static int user_zonelist_order = ZONELIST_ORDER_DEFAULT;
  2044. /* string for sysctl */
  2045. #define NUMA_ZONELIST_ORDER_LEN 16
  2046. char numa_zonelist_order[16] = "default";
  2047. /*
  2048. * interface for configure zonelist ordering.
  2049. * command line option "numa_zonelist_order"
  2050. * = "[dD]efault - default, automatic configuration.
  2051. * = "[nN]ode - order by node locality, then by zone within node
  2052. * = "[zZ]one - order by zone, then by locality within zone
  2053. */
  2054. static int __parse_numa_zonelist_order(char *s)
  2055. {
  2056. if (*s == 'd' || *s == 'D') {
  2057. user_zonelist_order = ZONELIST_ORDER_DEFAULT;
  2058. } else if (*s == 'n' || *s == 'N') {
  2059. user_zonelist_order = ZONELIST_ORDER_NODE;
  2060. } else if (*s == 'z' || *s == 'Z') {
  2061. user_zonelist_order = ZONELIST_ORDER_ZONE;
  2062. } else {
  2063. printk(KERN_WARNING
  2064. "Ignoring invalid numa_zonelist_order value: "
  2065. "%s\n", s);
  2066. return -EINVAL;
  2067. }
  2068. return 0;
  2069. }
  2070. static __init int setup_numa_zonelist_order(char *s)
  2071. {
  2072. if (s)
  2073. return __parse_numa_zonelist_order(s);
  2074. return 0;
  2075. }
  2076. early_param("numa_zonelist_order", setup_numa_zonelist_order);
  2077. /*
  2078. * sysctl handler for numa_zonelist_order
  2079. */
  2080. int numa_zonelist_order_handler(ctl_table *table, int write,
  2081. void __user *buffer, size_t *length,
  2082. loff_t *ppos)
  2083. {
  2084. char saved_string[NUMA_ZONELIST_ORDER_LEN];
  2085. int ret;
  2086. static DEFINE_MUTEX(zl_order_mutex);
  2087. mutex_lock(&zl_order_mutex);
  2088. if (write)
  2089. strcpy(saved_string, (char*)table->data);
  2090. ret = proc_dostring(table, write, buffer, length, ppos);
  2091. if (ret)
  2092. goto out;
  2093. if (write) {
  2094. int oldval = user_zonelist_order;
  2095. if (__parse_numa_zonelist_order((char*)table->data)) {
  2096. /*
  2097. * bogus value. restore saved string
  2098. */
  2099. strncpy((char*)table->data, saved_string,
  2100. NUMA_ZONELIST_ORDER_LEN);
  2101. user_zonelist_order = oldval;
  2102. } else if (oldval != user_zonelist_order)
  2103. build_all_zonelists();
  2104. }
  2105. out:
  2106. mutex_unlock(&zl_order_mutex);
  2107. return ret;
  2108. }
  2109. #define MAX_NODE_LOAD (nr_online_nodes)
  2110. static int node_load[MAX_NUMNODES];
  2111. /**
  2112. * find_next_best_node - find the next node that should appear in a given node's fallback list
  2113. * @node: node whose fallback list we're appending
  2114. * @used_node_mask: nodemask_t of already used nodes
  2115. *
  2116. * We use a number of factors to determine which is the next node that should
  2117. * appear on a given node's fallback list. The node should not have appeared
  2118. * already in @node's fallback list, and it should be the next closest node
  2119. * according to the distance array (which contains arbitrary distance values
  2120. * from each node to each node in the system), and should also prefer nodes
  2121. * with no CPUs, since presumably they'll have very little allocation pressure
  2122. * on them otherwise.
  2123. * It returns -1 if no node is found.
  2124. */
  2125. static int find_next_best_node(int node, nodemask_t *used_node_mask)
  2126. {
  2127. int n, val;
  2128. int min_val = INT_MAX;
  2129. int best_node = -1;
  2130. const struct cpumask *tmp = cpumask_of_node(0);
  2131. /* Use the local node if we haven't already */
  2132. if (!node_isset(node, *used_node_mask)) {
  2133. node_set(node, *used_node_mask);
  2134. return node;
  2135. }
  2136. for_each_node_state(n, N_HIGH_MEMORY) {
  2137. /* Don't want a node to appear more than once */
  2138. if (node_isset(n, *used_node_mask))
  2139. continue;
  2140. /* Use the distance array to find the distance */
  2141. val = node_distance(node, n);
  2142. /* Penalize nodes under us ("prefer the next node") */
  2143. val += (n < node);
  2144. /* Give preference to headless and unused nodes */
  2145. tmp = cpumask_of_node(n);
  2146. if (!cpumask_empty(tmp))
  2147. val += PENALTY_FOR_NODE_WITH_CPUS;
  2148. /* Slight preference for less loaded node */
  2149. val *= (MAX_NODE_LOAD*MAX_NUMNODES);
  2150. val += node_load[n];
  2151. if (val < min_val) {
  2152. min_val = val;
  2153. best_node = n;
  2154. }
  2155. }
  2156. if (best_node >= 0)
  2157. node_set(best_node, *used_node_mask);
  2158. return best_node;
  2159. }
  2160. /*
  2161. * Build zonelists ordered by node and zones within node.
  2162. * This results in maximum locality--normal zone overflows into local
  2163. * DMA zone, if any--but risks exhausting DMA zone.
  2164. */
  2165. static void build_zonelists_in_node_order(pg_data_t *pgdat, int node)
  2166. {
  2167. int j;
  2168. struct zonelist *zonelist;
  2169. zonelist = &pgdat->node_zonelists[0];
  2170. for (j = 0; zonelist->_zonerefs[j].zone != NULL; j++)
  2171. ;
  2172. j = build_zonelists_node(NODE_DATA(node), zonelist, j,
  2173. MAX_NR_ZONES - 1);
  2174. zonelist->_zonerefs[j].zone = NULL;
  2175. zonelist->_zonerefs[j].zone_idx = 0;
  2176. }
  2177. /*
  2178. * Build gfp_thisnode zonelists
  2179. */
  2180. static void build_thisnode_zonelists(pg_data_t *pgdat)
  2181. {
  2182. int j;
  2183. struct zonelist *zonelist;
  2184. zonelist = &pgdat->node_zonelists[1];
  2185. j = build_zonelists_node(pgdat, zonelist, 0, MAX_NR_ZONES - 1);
  2186. zonelist->_zonerefs[j].zone = NULL;
  2187. zonelist->_zonerefs[j].zone_idx = 0;
  2188. }
  2189. /*
  2190. * Build zonelists ordered by zone and nodes within zones.
  2191. * This results in conserving DMA zone[s] until all Normal memory is
  2192. * exhausted, but results in overflowing to remote node while memory
  2193. * may still exist in local DMA zone.
  2194. */
  2195. static int node_order[MAX_NUMNODES];
  2196. static void build_zonelists_in_zone_order(pg_data_t *pgdat, int nr_nodes)
  2197. {
  2198. int pos, j, node;
  2199. int zone_type; /* needs to be signed */
  2200. struct zone *z;
  2201. struct zonelist *zonelist;
  2202. zonelist = &pgdat->node_zonelists[0];
  2203. pos = 0;
  2204. for (zone_type = MAX_NR_ZONES - 1; zone_type >= 0; zone_type--) {
  2205. for (j = 0; j < nr_nodes; j++) {
  2206. node = node_order[j];
  2207. z = &NODE_DATA(node)->node_zones[zone_type];
  2208. if (populated_zone(z)) {
  2209. zoneref_set_zone(z,
  2210. &zonelist->_zonerefs[pos++]);
  2211. check_highest_zone(zone_type);
  2212. }
  2213. }
  2214. }
  2215. zonelist->_zonerefs[pos].zone = NULL;
  2216. zonelist->_zonerefs[pos].zone_idx = 0;
  2217. }
  2218. static int default_zonelist_order(void)
  2219. {
  2220. int nid, zone_type;
  2221. unsigned long low_kmem_size,total_size;
  2222. struct zone *z;
  2223. int average_size;
  2224. /*
  2225. * ZONE_DMA and ZONE_DMA32 can be very small area in the sytem.
  2226. * If they are really small and used heavily, the system can fall
  2227. * into OOM very easily.
  2228. * This function detect ZONE_DMA/DMA32 size and confgigures zone order.
  2229. */
  2230. /* Is there ZONE_NORMAL ? (ex. ppc has only DMA zone..) */
  2231. low_kmem_size = 0;
  2232. total_size = 0;
  2233. for_each_online_node(nid) {
  2234. for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
  2235. z = &NODE_DATA(nid)->node_zones[zone_type];
  2236. if (populated_zone(z)) {
  2237. if (zone_type < ZONE_NORMAL)
  2238. low_kmem_size += z->present_pages;
  2239. total_size += z->present_pages;
  2240. }
  2241. }
  2242. }
  2243. if (!low_kmem_size || /* there are no DMA area. */
  2244. low_kmem_size > total_size/2) /* DMA/DMA32 is big. */
  2245. return ZONELIST_ORDER_NODE;
  2246. /*
  2247. * look into each node's config.
  2248. * If there is a node whose DMA/DMA32 memory is very big area on
  2249. * local memory, NODE_ORDER may be suitable.
  2250. */
  2251. average_size = total_size /
  2252. (nodes_weight(node_states[N_HIGH_MEMORY]) + 1);
  2253. for_each_online_node(nid) {
  2254. low_kmem_size = 0;
  2255. total_size = 0;
  2256. for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
  2257. z = &NODE_DATA(nid)->node_zones[zone_type];
  2258. if (populated_zone(z)) {
  2259. if (zone_type < ZONE_NORMAL)
  2260. low_kmem_size += z->present_pages;
  2261. total_size += z->present_pages;
  2262. }
  2263. }
  2264. if (low_kmem_size &&
  2265. total_size > average_size && /* ignore small node */
  2266. low_kmem_size > total_size * 70/100)
  2267. return ZONELIST_ORDER_NODE;
  2268. }
  2269. return ZONELIST_ORDER_ZONE;
  2270. }
  2271. static void set_zonelist_order(void)
  2272. {
  2273. if (user_zonelist_order == ZONELIST_ORDER_DEFAULT)
  2274. current_zonelist_order = default_zonelist_order();
  2275. else
  2276. current_zonelist_order = user_zonelist_order;
  2277. }
  2278. static void build_zonelists(pg_data_t *pgdat)
  2279. {
  2280. int j, node, load;
  2281. enum zone_type i;
  2282. nodemask_t used_mask;
  2283. int local_node, prev_node;
  2284. struct zonelist *zonelist;
  2285. int order = current_zonelist_order;
  2286. /* initialize zonelists */
  2287. for (i = 0; i < MAX_ZONELISTS; i++) {
  2288. zonelist = pgdat->node_zonelists + i;
  2289. zonelist->_zonerefs[0].zone = NULL;
  2290. zonelist->_zonerefs[0].zone_idx = 0;
  2291. }
  2292. /* NUMA-aware ordering of nodes */
  2293. local_node = pgdat->node_id;
  2294. load = nr_online_nodes;
  2295. prev_node = local_node;
  2296. nodes_clear(used_mask);
  2297. memset(node_order, 0, sizeof(node_order));
  2298. j = 0;
  2299. while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
  2300. int distance = node_distance(local_node, node);
  2301. /*
  2302. * If another node is sufficiently far away then it is better
  2303. * to reclaim pages in a zone before going off node.
  2304. */
  2305. if (distance > RECLAIM_DISTANCE)
  2306. zone_reclaim_mode = 1;
  2307. /*
  2308. * We don't want to pressure a particular node.
  2309. * So adding penalty to the first node in same
  2310. * distance group to make it round-robin.
  2311. */
  2312. if (distance != node_distance(local_node, prev_node))
  2313. node_load[node] = load;
  2314. prev_node = node;
  2315. load--;
  2316. if (order == ZONELIST_ORDER_NODE)
  2317. build_zonelists_in_node_order(pgdat, node);
  2318. else
  2319. node_order[j++] = node; /* remember order */
  2320. }
  2321. if (order == ZONELIST_ORDER_ZONE) {
  2322. /* calculate node order -- i.e., DMA last! */
  2323. build_zonelists_in_zone_order(pgdat, j);
  2324. }
  2325. build_thisnode_zonelists(pgdat);
  2326. }
  2327. /* Construct the zonelist performance cache - see further mmzone.h */
  2328. static void build_zonelist_cache(pg_data_t *pgdat)
  2329. {
  2330. struct zonelist *zonelist;
  2331. struct zonelist_cache *zlc;
  2332. struct zoneref *z;
  2333. zonelist = &pgdat->node_zonelists[0];
  2334. zonelist->zlcache_ptr = zlc = &zonelist->zlcache;
  2335. bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
  2336. for (z = zonelist->_zonerefs; z->zone; z++)
  2337. zlc->z_to_n[z - zonelist->_zonerefs] = zonelist_node_idx(z);
  2338. }
  2339. #else /* CONFIG_NUMA */
  2340. static void set_zonelist_order(void)
  2341. {
  2342. current_zonelist_order = ZONELIST_ORDER_ZONE;
  2343. }
  2344. static void build_zonelists(pg_data_t *pgdat)
  2345. {
  2346. int node, local_node;
  2347. enum zone_type j;
  2348. struct zonelist *zonelist;
  2349. local_node = pgdat->node_id;
  2350. zonelist = &pgdat->node_zonelists[0];
  2351. j = build_zonelists_node(pgdat, zonelist, 0, MAX_NR_ZONES - 1);
  2352. /*
  2353. * Now we build the zonelist so that it contains the zones
  2354. * of all the other nodes.
  2355. * We don't want to pressure a particular node, so when
  2356. * building the zones for node N, we make sure that the
  2357. * zones coming right after the local ones are those from
  2358. * node N+1 (modulo N)
  2359. */
  2360. for (node = local_node + 1; node < MAX_NUMNODES; node++) {
  2361. if (!node_online(node))
  2362. continue;
  2363. j = build_zonelists_node(NODE_DATA(node), zonelist, j,
  2364. MAX_NR_ZONES - 1);
  2365. }
  2366. for (node = 0; node < local_node; node++) {
  2367. if (!node_online(node))
  2368. continue;
  2369. j = build_zonelists_node(NODE_DATA(node), zonelist, j,
  2370. MAX_NR_ZONES - 1);
  2371. }
  2372. zonelist->_zonerefs[j].zone = NULL;
  2373. zonelist->_zonerefs[j].zone_idx = 0;
  2374. }
  2375. /* non-NUMA variant of zonelist performance cache - just NULL zlcache_ptr */
  2376. static void build_zonelist_cache(pg_data_t *pgdat)
  2377. {
  2378. pgdat->node_zonelists[0].zlcache_ptr = NULL;
  2379. }
  2380. #endif /* CONFIG_NUMA */
  2381. /*
  2382. * Boot pageset table. One per cpu which is going to be used for all
  2383. * zones and all nodes. The parameters will be set in such a way
  2384. * that an item put on a list will immediately be handed over to
  2385. * the buddy list. This is safe since pageset manipulation is done
  2386. * with interrupts disabled.
  2387. *
  2388. * The boot_pagesets must be kept even after bootup is complete for
  2389. * unused processors and/or zones. They do play a role for bootstrapping
  2390. * hotplugged processors.
  2391. *
  2392. * zoneinfo_show() and maybe other functions do
  2393. * not check if the processor is online before following the pageset pointer.
  2394. * Other parts of the kernel may not check if the zone is available.
  2395. */
  2396. static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
  2397. static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
  2398. /* return values int ....just for stop_machine() */
  2399. static int __build_all_zonelists(void *dummy)
  2400. {
  2401. int nid;
  2402. int cpu;
  2403. #ifdef CONFIG_NUMA
  2404. memset(node_load, 0, sizeof(node_load));
  2405. #endif
  2406. for_each_online_node(nid) {
  2407. pg_data_t *pgdat = NODE_DATA(nid);
  2408. build_zonelists(pgdat);
  2409. build_zonelist_cache(pgdat);
  2410. }
  2411. /*
  2412. * Initialize the boot_pagesets that are going to be used
  2413. * for bootstrapping processors. The real pagesets for
  2414. * each zone will be allocated later when the per cpu
  2415. * allocator is available.
  2416. *
  2417. * boot_pagesets are used also for bootstrapping offline
  2418. * cpus if the system is already booted because the pagesets
  2419. * are needed to initialize allocators on a specific cpu too.
  2420. * F.e. the percpu allocator needs the page allocator which
  2421. * needs the percpu allocator in order to allocate its pagesets
  2422. * (a chicken-egg dilemma).
  2423. */
  2424. for_each_possible_cpu(cpu)
  2425. setup_pageset(&per_cpu(boot_pageset, cpu), 0);
  2426. return 0;
  2427. }
  2428. void build_all_zonelists(void)
  2429. {
  2430. set_zonelist_order();
  2431. if (system_state == SYSTEM_BOOTING) {
  2432. __build_all_zonelists(NULL);
  2433. mminit_verify_zonelist();
  2434. cpuset_init_current_mems_allowed();
  2435. } else {
  2436. /* we have to stop all cpus to guarantee there is no user
  2437. of zonelist */
  2438. stop_machine(__build_all_zonelists, NULL, NULL);
  2439. /* cpuset refresh routine should be here */
  2440. }
  2441. vm_total_pages = nr_free_pagecache_pages();
  2442. /*
  2443. * Disable grouping by mobility if the number of pages in the
  2444. * system is too low to allow the mechanism to work. It would be
  2445. * more accurate, but expensive to check per-zone. This check is
  2446. * made on memory-hotadd so a system can start with mobility
  2447. * disabled and enable it later
  2448. */
  2449. if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
  2450. page_group_by_mobility_disabled = 1;
  2451. else
  2452. page_group_by_mobility_disabled = 0;
  2453. printk("Built %i zonelists in %s order, mobility grouping %s. "
  2454. "Total pages: %ld\n",
  2455. nr_online_nodes,
  2456. zonelist_order_name[current_zonelist_order],
  2457. page_group_by_mobility_disabled ? "off" : "on",
  2458. vm_total_pages);
  2459. #ifdef CONFIG_NUMA
  2460. printk("Policy zone: %s\n", zone_names[policy_zone]);
  2461. #endif
  2462. }
  2463. /*
  2464. * Helper functions to size the waitqueue hash table.
  2465. * Essentially these want to choose hash table sizes sufficiently
  2466. * large so that collisions trying to wait on pages are rare.
  2467. * But in fact, the number of active page waitqueues on typical
  2468. * systems is ridiculously low, less than 200. So this is even
  2469. * conservative, even though it seems large.
  2470. *
  2471. * The constant PAGES_PER_WAITQUEUE specifies the ratio of pages to
  2472. * waitqueues, i.e. the size of the waitq table given the number of pages.
  2473. */
  2474. #define PAGES_PER_WAITQUEUE 256
  2475. #ifndef CONFIG_MEMORY_HOTPLUG
  2476. static inline unsigned long wait_table_hash_nr_entries(unsigned long pages)
  2477. {
  2478. unsigned long size = 1;
  2479. pages /= PAGES_PER_WAITQUEUE;
  2480. while (size < pages)
  2481. size <<= 1;
  2482. /*
  2483. * Once we have dozens or even hundreds of threads sleeping
  2484. * on IO we've got bigger problems than wait queue collision.
  2485. * Limit the size of the wait table to a reasonable size.
  2486. */
  2487. size = min(size, 4096UL);
  2488. return max(size, 4UL);
  2489. }
  2490. #else
  2491. /*
  2492. * A zone's size might be changed by hot-add, so it is not possible to determine
  2493. * a suitable size for its wait_table. So we use the maximum size now.
  2494. *
  2495. * The max wait table size = 4096 x sizeof(wait_queue_head_t). ie:
  2496. *
  2497. * i386 (preemption config) : 4096 x 16 = 64Kbyte.
  2498. * ia64, x86-64 (no preemption): 4096 x 20 = 80Kbyte.
  2499. * ia64, x86-64 (preemption) : 4096 x 24 = 96Kbyte.
  2500. *
  2501. * The maximum entries are prepared when a zone's memory is (512K + 256) pages
  2502. * or more by the traditional way. (See above). It equals:
  2503. *
  2504. * i386, x86-64, powerpc(4K page size) : = ( 2G + 1M)byte.
  2505. * ia64(16K page size) : = ( 8G + 4M)byte.
  2506. * powerpc (64K page size) : = (32G +16M)byte.
  2507. */
  2508. static inline unsigned long wait_table_hash_nr_entries(unsigned long pages)
  2509. {
  2510. return 4096UL;
  2511. }
  2512. #endif
  2513. /*
  2514. * This is an integer logarithm so that shifts can be used later
  2515. * to extract the more random high bits from the multiplicative
  2516. * hash function before the remainder is taken.
  2517. */
  2518. static inline unsigned long wait_table_bits(unsigned long size)
  2519. {
  2520. return ffz(~size);
  2521. }
  2522. #define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1))
  2523. /*
  2524. * Mark a number of pageblocks as MIGRATE_RESERVE. The number
  2525. * of blocks reserved is based on min_wmark_pages(zone). The memory within
  2526. * the reserve will tend to store contiguous free pages. Setting min_free_kbytes
  2527. * higher will lead to a bigger reserve which will get freed as contiguous
  2528. * blocks as reclaim kicks in
  2529. */
  2530. static void setup_zone_migrate_reserve(struct zone *zone)
  2531. {
  2532. unsigned long start_pfn, pfn, end_pfn;
  2533. struct page *page;
  2534. unsigned long block_migratetype;
  2535. int reserve;
  2536. /* Get the start pfn, end pfn and the number of blocks to reserve */
  2537. start_pfn = zone->zone_start_pfn;
  2538. end_pfn = start_pfn + zone->spanned_pages;
  2539. reserve = roundup(min_wmark_pages(zone), pageblock_nr_pages) >>
  2540. pageblock_order;
  2541. /*
  2542. * Reserve blocks are generally in place to help high-order atomic
  2543. * allocations that are short-lived. A min_free_kbytes value that
  2544. * would result in more than 2 reserve blocks for atomic allocations
  2545. * is assumed to be in place to help anti-fragmentation for the
  2546. * future allocation of hugepages at runtime.
  2547. */
  2548. reserve = min(2, reserve);
  2549. for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
  2550. if (!pfn_valid(pfn))
  2551. continue;
  2552. page = pfn_to_page(pfn);
  2553. /* Watch out for overlapping nodes */
  2554. if (page_to_nid(page) != zone_to_nid(zone))
  2555. continue;
  2556. /* Blocks with reserved pages will never free, skip them. */
  2557. if (PageReserved(page))
  2558. continue;
  2559. block_migratetype = get_pageblock_migratetype(page);
  2560. /* If this block is reserved, account for it */
  2561. if (reserve > 0 && block_migratetype == MIGRATE_RESERVE) {
  2562. reserve--;
  2563. continue;
  2564. }
  2565. /* Suitable for reserving if this block is movable */
  2566. if (reserve > 0 && block_migratetype == MIGRATE_MOVABLE) {
  2567. set_pageblock_migratetype(page, MIGRATE_RESERVE);
  2568. move_freepages_block(zone, page, MIGRATE_RESERVE);
  2569. reserve--;
  2570. continue;
  2571. }
  2572. /*
  2573. * If the reserve is met and this is a previous reserved block,
  2574. * take it back
  2575. */
  2576. if (block_migratetype == MIGRATE_RESERVE) {
  2577. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  2578. move_freepages_block(zone, page, MIGRATE_MOVABLE);
  2579. }
  2580. }
  2581. }
  2582. /*
  2583. * Initially all pages are reserved - free ones are freed
  2584. * up by free_all_bootmem() once the early boot process is
  2585. * done. Non-atomic initialization, single-pass.
  2586. */
  2587. void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
  2588. unsigned long start_pfn, enum memmap_context context)
  2589. {
  2590. struct page *page;
  2591. unsigned long end_pfn = start_pfn + size;
  2592. unsigned long pfn;
  2593. struct zone *z;
  2594. if (highest_memmap_pfn < end_pfn - 1)
  2595. highest_memmap_pfn = end_pfn - 1;
  2596. z = &NODE_DATA(nid)->node_zones[zone];
  2597. for (pfn = start_pfn; pfn < end_pfn; pfn++) {
  2598. /*
  2599. * There can be holes in boot-time mem_map[]s
  2600. * handed to this function. They do not
  2601. * exist on hotplugged memory.
  2602. */
  2603. if (context == MEMMAP_EARLY) {
  2604. if (!early_pfn_valid(pfn))
  2605. continue;
  2606. if (!early_pfn_in_nid(pfn, nid))
  2607. continue;
  2608. }
  2609. page = pfn_to_page(pfn);
  2610. set_page_links(page, zone, nid, pfn);
  2611. mminit_verify_page_links(page, zone, nid, pfn);
  2612. init_page_count(page);
  2613. reset_page_mapcount(page);
  2614. SetPageReserved(page);
  2615. /*
  2616. * Mark the block movable so that blocks are reserved for
  2617. * movable at startup. This will force kernel allocations
  2618. * to reserve their blocks rather than leaking throughout
  2619. * the address space during boot when many long-lived
  2620. * kernel allocations are made. Later some blocks near
  2621. * the start are marked MIGRATE_RESERVE by
  2622. * setup_zone_migrate_reserve()
  2623. *
  2624. * bitmap is created for zone's valid pfn range. but memmap
  2625. * can be created for invalid pages (for alignment)
  2626. * check here not to call set_pageblock_migratetype() against
  2627. * pfn out of zone.
  2628. */
  2629. if ((z->zone_start_pfn <= pfn)
  2630. && (pfn < z->zone_start_pfn + z->spanned_pages)
  2631. && !(pfn & (pageblock_nr_pages - 1)))
  2632. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  2633. INIT_LIST_HEAD(&page->lru);
  2634. #ifdef WANT_PAGE_VIRTUAL
  2635. /* The shift won't overflow because ZONE_NORMAL is below 4G. */
  2636. if (!is_highmem_idx(zone))
  2637. set_page_address(page, __va(pfn << PAGE_SHIFT));
  2638. #endif
  2639. }
  2640. }
  2641. static void __meminit zone_init_free_lists(struct zone *zone)
  2642. {
  2643. int order, t;
  2644. for_each_migratetype_order(order, t) {
  2645. INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
  2646. zone->free_area[order].nr_free = 0;
  2647. }
  2648. }
  2649. #ifndef __HAVE_ARCH_MEMMAP_INIT
  2650. #define memmap_init(size, nid, zone, start_pfn) \
  2651. memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY)
  2652. #endif
  2653. static int zone_batchsize(struct zone *zone)
  2654. {
  2655. #ifdef CONFIG_MMU
  2656. int batch;
  2657. /*
  2658. * The per-cpu-pages pools are set to around 1000th of the
  2659. * size of the zone. But no more than 1/2 of a meg.
  2660. *
  2661. * OK, so we don't know how big the cache is. So guess.
  2662. */
  2663. batch = zone->present_pages / 1024;
  2664. if (batch * PAGE_SIZE > 512 * 1024)
  2665. batch = (512 * 1024) / PAGE_SIZE;
  2666. batch /= 4; /* We effectively *= 4 below */
  2667. if (batch < 1)
  2668. batch = 1;
  2669. /*
  2670. * Clamp the batch to a 2^n - 1 value. Having a power
  2671. * of 2 value was found to be more likely to have
  2672. * suboptimal cache aliasing properties in some cases.
  2673. *
  2674. * For example if 2 tasks are alternately allocating
  2675. * batches of pages, one task can end up with a lot
  2676. * of pages of one half of the possible page colors
  2677. * and the other with pages of the other colors.
  2678. */
  2679. batch = rounddown_pow_of_two(batch + batch/2) - 1;
  2680. return batch;
  2681. #else
  2682. /* The deferral and batching of frees should be suppressed under NOMMU
  2683. * conditions.
  2684. *
  2685. * The problem is that NOMMU needs to be able to allocate large chunks
  2686. * of contiguous memory as there's no hardware page translation to
  2687. * assemble apparent contiguous memory from discontiguous pages.
  2688. *
  2689. * Queueing large contiguous runs of pages for batching, however,
  2690. * causes the pages to actually be freed in smaller chunks. As there
  2691. * can be a significant delay between the individual batches being
  2692. * recycled, this leads to the once large chunks of space being
  2693. * fragmented and becoming unavailable for high-order allocations.
  2694. */
  2695. return 0;
  2696. #endif
  2697. }
  2698. static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
  2699. {
  2700. struct per_cpu_pages *pcp;
  2701. int migratetype;
  2702. memset(p, 0, sizeof(*p));
  2703. pcp = &p->pcp;
  2704. pcp->count = 0;
  2705. pcp->high = 6 * batch;
  2706. pcp->batch = max(1UL, 1 * batch);
  2707. for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++)
  2708. INIT_LIST_HEAD(&pcp->lists[migratetype]);
  2709. }
  2710. /*
  2711. * setup_pagelist_highmark() sets the high water mark for hot per_cpu_pagelist
  2712. * to the value high for the pageset p.
  2713. */
  2714. static void setup_pagelist_highmark(struct per_cpu_pageset *p,
  2715. unsigned long high)
  2716. {
  2717. struct per_cpu_pages *pcp;
  2718. pcp = &p->pcp;
  2719. pcp->high = high;
  2720. pcp->batch = max(1UL, high/4);
  2721. if ((high/4) > (PAGE_SHIFT * 8))
  2722. pcp->batch = PAGE_SHIFT * 8;
  2723. }
  2724. /*
  2725. * Allocate per cpu pagesets and initialize them.
  2726. * Before this call only boot pagesets were available.
  2727. * Boot pagesets will no longer be used by this processorr
  2728. * after setup_per_cpu_pageset().
  2729. */
  2730. void __init setup_per_cpu_pageset(void)
  2731. {
  2732. struct zone *zone;
  2733. int cpu;
  2734. for_each_populated_zone(zone) {
  2735. zone->pageset = alloc_percpu(struct per_cpu_pageset);
  2736. for_each_possible_cpu(cpu) {
  2737. struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);
  2738. setup_pageset(pcp, zone_batchsize(zone));
  2739. if (percpu_pagelist_fraction)
  2740. setup_pagelist_highmark(pcp,
  2741. (zone->present_pages /
  2742. percpu_pagelist_fraction));
  2743. }
  2744. }
  2745. }
  2746. static noinline __init_refok
  2747. int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages)
  2748. {
  2749. int i;
  2750. struct pglist_data *pgdat = zone->zone_pgdat;
  2751. size_t alloc_size;
  2752. /*
  2753. * The per-page waitqueue mechanism uses hashed waitqueues
  2754. * per zone.
  2755. */
  2756. zone->wait_table_hash_nr_entries =
  2757. wait_table_hash_nr_entries(zone_size_pages);
  2758. zone->wait_table_bits =
  2759. wait_table_bits(zone->wait_table_hash_nr_entries);
  2760. alloc_size = zone->wait_table_hash_nr_entries
  2761. * sizeof(wait_queue_head_t);
  2762. if (!slab_is_available()) {
  2763. zone->wait_table = (wait_queue_head_t *)
  2764. alloc_bootmem_node(pgdat, alloc_size);
  2765. } else {
  2766. /*
  2767. * This case means that a zone whose size was 0 gets new memory
  2768. * via memory hot-add.
  2769. * But it may be the case that a new node was hot-added. In
  2770. * this case vmalloc() will not be able to use this new node's
  2771. * memory - this wait_table must be initialized to use this new
  2772. * node itself as well.
  2773. * To use this new node's memory, further consideration will be
  2774. * necessary.
  2775. */
  2776. zone->wait_table = vmalloc(alloc_size);
  2777. }
  2778. if (!zone->wait_table)
  2779. return -ENOMEM;
  2780. for(i = 0; i < zone->wait_table_hash_nr_entries; ++i)
  2781. init_waitqueue_head(zone->wait_table + i);
  2782. return 0;
  2783. }
  2784. static int __zone_pcp_update(void *data)
  2785. {
  2786. struct zone *zone = data;
  2787. int cpu;
  2788. unsigned long batch = zone_batchsize(zone), flags;
  2789. for_each_possible_cpu(cpu) {
  2790. struct per_cpu_pageset *pset;
  2791. struct per_cpu_pages *pcp;
  2792. pset = per_cpu_ptr(zone->pageset, cpu);
  2793. pcp = &pset->pcp;
  2794. local_irq_save(flags);
  2795. free_pcppages_bulk(zone, pcp->count, pcp);
  2796. setup_pageset(pset, batch);
  2797. local_irq_restore(flags);
  2798. }
  2799. return 0;
  2800. }
  2801. void zone_pcp_update(struct zone *zone)
  2802. {
  2803. stop_machine(__zone_pcp_update, zone, NULL);
  2804. }
  2805. static __meminit void zone_pcp_init(struct zone *zone)
  2806. {
  2807. /*
  2808. * per cpu subsystem is not up at this point. The following code
  2809. * relies on the ability of the linker to provide the
  2810. * offset of a (static) per cpu variable into the per cpu area.
  2811. */
  2812. zone->pageset = &boot_pageset;
  2813. if (zone->present_pages)
  2814. printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n",
  2815. zone->name, zone->present_pages,
  2816. zone_batchsize(zone));
  2817. }
  2818. __meminit int init_currently_empty_zone(struct zone *zone,
  2819. unsigned long zone_start_pfn,
  2820. unsigned long size,
  2821. enum memmap_context context)
  2822. {
  2823. struct pglist_data *pgdat = zone->zone_pgdat;
  2824. int ret;
  2825. ret = zone_wait_table_init(zone, size);
  2826. if (ret)
  2827. return ret;
  2828. pgdat->nr_zones = zone_idx(zone) + 1;
  2829. zone->zone_start_pfn = zone_start_pfn;
  2830. mminit_dprintk(MMINIT_TRACE, "memmap_init",
  2831. "Initialising map node %d zone %lu pfns %lu -> %lu\n",
  2832. pgdat->node_id,
  2833. (unsigned long)zone_idx(zone),
  2834. zone_start_pfn, (zone_start_pfn + size));
  2835. zone_init_free_lists(zone);
  2836. return 0;
  2837. }
  2838. #ifdef CONFIG_ARCH_POPULATES_NODE_MAP
  2839. /*
  2840. * Basic iterator support. Return the first range of PFNs for a node
  2841. * Note: nid == MAX_NUMNODES returns first region regardless of node
  2842. */
  2843. static int __meminit first_active_region_index_in_nid(int nid)
  2844. {
  2845. int i;
  2846. for (i = 0; i < nr_nodemap_entries; i++)
  2847. if (nid == MAX_NUMNODES || early_node_map[i].nid == nid)
  2848. return i;
  2849. return -1;
  2850. }
  2851. /*
  2852. * Basic iterator support. Return the next active range of PFNs for a node
  2853. * Note: nid == MAX_NUMNODES returns next region regardless of node
  2854. */
  2855. static int __meminit next_active_region_index_in_nid(int index, int nid)
  2856. {
  2857. for (index = index + 1; index < nr_nodemap_entries; index++)
  2858. if (nid == MAX_NUMNODES || early_node_map[index].nid == nid)
  2859. return index;
  2860. return -1;
  2861. }
  2862. #ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
  2863. /*
  2864. * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
  2865. * Architectures may implement their own version but if add_active_range()
  2866. * was used and there are no special requirements, this is a convenient
  2867. * alternative
  2868. */
  2869. int __meminit __early_pfn_to_nid(unsigned long pfn)
  2870. {
  2871. int i;
  2872. for (i = 0; i < nr_nodemap_entries; i++) {
  2873. unsigned long start_pfn = early_node_map[i].start_pfn;
  2874. unsigned long end_pfn = early_node_map[i].end_pfn;
  2875. if (start_pfn <= pfn && pfn < end_pfn)
  2876. return early_node_map[i].nid;
  2877. }
  2878. /* This is a memory hole */
  2879. return -1;
  2880. }
  2881. #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
  2882. int __meminit early_pfn_to_nid(unsigned long pfn)
  2883. {
  2884. int nid;
  2885. nid = __early_pfn_to_nid(pfn);
  2886. if (nid >= 0)
  2887. return nid;
  2888. /* just returns 0 */
  2889. return 0;
  2890. }
  2891. #ifdef CONFIG_NODES_SPAN_OTHER_NODES
  2892. bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
  2893. {
  2894. int nid;
  2895. nid = __early_pfn_to_nid(pfn);
  2896. if (nid >= 0 && nid != node)
  2897. return false;
  2898. return true;
  2899. }
  2900. #endif
  2901. /* Basic iterator support to walk early_node_map[] */
  2902. #define for_each_active_range_index_in_nid(i, nid) \
  2903. for (i = first_active_region_index_in_nid(nid); i != -1; \
  2904. i = next_active_region_index_in_nid(i, nid))
  2905. /**
  2906. * free_bootmem_with_active_regions - Call free_bootmem_node for each active range
  2907. * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed.
  2908. * @max_low_pfn: The highest PFN that will be passed to free_bootmem_node
  2909. *
  2910. * If an architecture guarantees that all ranges registered with
  2911. * add_active_ranges() contain no holes and may be freed, this
  2912. * this function may be used instead of calling free_bootmem() manually.
  2913. */
  2914. void __init free_bootmem_with_active_regions(int nid,
  2915. unsigned long max_low_pfn)
  2916. {
  2917. int i;
  2918. for_each_active_range_index_in_nid(i, nid) {
  2919. unsigned long size_pages = 0;
  2920. unsigned long end_pfn = early_node_map[i].end_pfn;
  2921. if (early_node_map[i].start_pfn >= max_low_pfn)
  2922. continue;
  2923. if (end_pfn > max_low_pfn)
  2924. end_pfn = max_low_pfn;
  2925. size_pages = end_pfn - early_node_map[i].start_pfn;
  2926. free_bootmem_node(NODE_DATA(early_node_map[i].nid),
  2927. PFN_PHYS(early_node_map[i].start_pfn),
  2928. size_pages << PAGE_SHIFT);
  2929. }
  2930. }
  2931. int __init add_from_early_node_map(struct range *range, int az,
  2932. int nr_range, int nid)
  2933. {
  2934. int i;
  2935. u64 start, end;
  2936. /* need to go over early_node_map to find out good range for node */
  2937. for_each_active_range_index_in_nid(i, nid) {
  2938. start = early_node_map[i].start_pfn;
  2939. end = early_node_map[i].end_pfn;
  2940. nr_range = add_range(range, az, nr_range, start, end);
  2941. }
  2942. return nr_range;
  2943. }
  2944. #ifdef CONFIG_NO_BOOTMEM
  2945. void * __init __alloc_memory_core_early(int nid, u64 size, u64 align,
  2946. u64 goal, u64 limit)
  2947. {
  2948. int i;
  2949. void *ptr;
  2950. /* need to go over early_node_map to find out good range for node */
  2951. for_each_active_range_index_in_nid(i, nid) {
  2952. u64 addr;
  2953. u64 ei_start, ei_last;
  2954. ei_last = early_node_map[i].end_pfn;
  2955. ei_last <<= PAGE_SHIFT;
  2956. ei_start = early_node_map[i].start_pfn;
  2957. ei_start <<= PAGE_SHIFT;
  2958. addr = find_early_area(ei_start, ei_last,
  2959. goal, limit, size, align);
  2960. if (addr == -1ULL)
  2961. continue;
  2962. #if 0
  2963. printk(KERN_DEBUG "alloc (nid=%d %llx - %llx) (%llx - %llx) %llx %llx => %llx\n",
  2964. nid,
  2965. ei_start, ei_last, goal, limit, size,
  2966. align, addr);
  2967. #endif
  2968. ptr = phys_to_virt(addr);
  2969. memset(ptr, 0, size);
  2970. reserve_early_without_check(addr, addr + size, "BOOTMEM");
  2971. return ptr;
  2972. }
  2973. return NULL;
  2974. }
  2975. #endif
  2976. void __init work_with_active_regions(int nid, work_fn_t work_fn, void *data)
  2977. {
  2978. int i;
  2979. int ret;
  2980. for_each_active_range_index_in_nid(i, nid) {
  2981. ret = work_fn(early_node_map[i].start_pfn,
  2982. early_node_map[i].end_pfn, data);
  2983. if (ret)
  2984. break;
  2985. }
  2986. }
  2987. /**
  2988. * sparse_memory_present_with_active_regions - Call memory_present for each active range
  2989. * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used.
  2990. *
  2991. * If an architecture guarantees that all ranges registered with
  2992. * add_active_ranges() contain no holes and may be freed, this
  2993. * function may be used instead of calling memory_present() manually.
  2994. */
  2995. void __init sparse_memory_present_with_active_regions(int nid)
  2996. {
  2997. int i;
  2998. for_each_active_range_index_in_nid(i, nid)
  2999. memory_present(early_node_map[i].nid,
  3000. early_node_map[i].start_pfn,
  3001. early_node_map[i].end_pfn);
  3002. }
  3003. /**
  3004. * get_pfn_range_for_nid - Return the start and end page frames for a node
  3005. * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
  3006. * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
  3007. * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
  3008. *
  3009. * It returns the start and end page frame of a node based on information
  3010. * provided by an arch calling add_active_range(). If called for a node
  3011. * with no available memory, a warning is printed and the start and end
  3012. * PFNs will be 0.
  3013. */
  3014. void __meminit get_pfn_range_for_nid(unsigned int nid,
  3015. unsigned long *start_pfn, unsigned long *end_pfn)
  3016. {
  3017. int i;
  3018. *start_pfn = -1UL;
  3019. *end_pfn = 0;
  3020. for_each_active_range_index_in_nid(i, nid) {
  3021. *start_pfn = min(*start_pfn, early_node_map[i].start_pfn);
  3022. *end_pfn = max(*end_pfn, early_node_map[i].end_pfn);
  3023. }
  3024. if (*start_pfn == -1UL)
  3025. *start_pfn = 0;
  3026. }
  3027. /*
  3028. * This finds a zone that can be used for ZONE_MOVABLE pages. The
  3029. * assumption is made that zones within a node are ordered in monotonic
  3030. * increasing memory addresses so that the "highest" populated zone is used
  3031. */
  3032. static void __init find_usable_zone_for_movable(void)
  3033. {
  3034. int zone_index;
  3035. for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
  3036. if (zone_index == ZONE_MOVABLE)
  3037. continue;
  3038. if (arch_zone_highest_possible_pfn[zone_index] >
  3039. arch_zone_lowest_possible_pfn[zone_index])
  3040. break;
  3041. }
  3042. VM_BUG_ON(zone_index == -1);
  3043. movable_zone = zone_index;
  3044. }
  3045. /*
  3046. * The zone ranges provided by the architecture do not include ZONE_MOVABLE
  3047. * because it is sized independant of architecture. Unlike the other zones,
  3048. * the starting point for ZONE_MOVABLE is not fixed. It may be different
  3049. * in each node depending on the size of each node and how evenly kernelcore
  3050. * is distributed. This helper function adjusts the zone ranges
  3051. * provided by the architecture for a given node by using the end of the
  3052. * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
  3053. * zones within a node are in order of monotonic increases memory addresses
  3054. */
  3055. static void __meminit adjust_zone_range_for_zone_movable(int nid,
  3056. unsigned long zone_type,
  3057. unsigned long node_start_pfn,
  3058. unsigned long node_end_pfn,
  3059. unsigned long *zone_start_pfn,
  3060. unsigned long *zone_end_pfn)
  3061. {
  3062. /* Only adjust if ZONE_MOVABLE is on this node */
  3063. if (zone_movable_pfn[nid]) {
  3064. /* Size ZONE_MOVABLE */
  3065. if (zone_type == ZONE_MOVABLE) {
  3066. *zone_start_pfn = zone_movable_pfn[nid];
  3067. *zone_end_pfn = min(node_end_pfn,
  3068. arch_zone_highest_possible_pfn[movable_zone]);
  3069. /* Adjust for ZONE_MOVABLE starting within this range */
  3070. } else if (*zone_start_pfn < zone_movable_pfn[nid] &&
  3071. *zone_end_pfn > zone_movable_pfn[nid]) {
  3072. *zone_end_pfn = zone_movable_pfn[nid];
  3073. /* Check if this whole range is within ZONE_MOVABLE */
  3074. } else if (*zone_start_pfn >= zone_movable_pfn[nid])
  3075. *zone_start_pfn = *zone_end_pfn;
  3076. }
  3077. }
  3078. /*
  3079. * Return the number of pages a zone spans in a node, including holes
  3080. * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
  3081. */
  3082. static unsigned long __meminit zone_spanned_pages_in_node(int nid,
  3083. unsigned long zone_type,
  3084. unsigned long *ignored)
  3085. {
  3086. unsigned long node_start_pfn, node_end_pfn;
  3087. unsigned long zone_start_pfn, zone_end_pfn;
  3088. /* Get the start and end of the node and zone */
  3089. get_pfn_range_for_nid(nid, &node_start_pfn, &node_end_pfn);
  3090. zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type];
  3091. zone_end_pfn = arch_zone_highest_possible_pfn[zone_type];
  3092. adjust_zone_range_for_zone_movable(nid, zone_type,
  3093. node_start_pfn, node_end_pfn,
  3094. &zone_start_pfn, &zone_end_pfn);
  3095. /* Check that this node has pages within the zone's required range */
  3096. if (zone_end_pfn < node_start_pfn || zone_start_pfn > node_end_pfn)
  3097. return 0;
  3098. /* Move the zone boundaries inside the node if necessary */
  3099. zone_end_pfn = min(zone_end_pfn, node_end_pfn);
  3100. zone_start_pfn = max(zone_start_pfn, node_start_pfn);
  3101. /* Return the spanned pages */
  3102. return zone_end_pfn - zone_start_pfn;
  3103. }
  3104. /*
  3105. * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
  3106. * then all holes in the requested range will be accounted for.
  3107. */
  3108. unsigned long __meminit __absent_pages_in_range(int nid,
  3109. unsigned long range_start_pfn,
  3110. unsigned long range_end_pfn)
  3111. {
  3112. int i = 0;
  3113. unsigned long prev_end_pfn = 0, hole_pages = 0;
  3114. unsigned long start_pfn;
  3115. /* Find the end_pfn of the first active range of pfns in the node */
  3116. i = first_active_region_index_in_nid(nid);
  3117. if (i == -1)
  3118. return 0;
  3119. prev_end_pfn = min(early_node_map[i].start_pfn, range_end_pfn);
  3120. /* Account for ranges before physical memory on this node */
  3121. if (early_node_map[i].start_pfn > range_start_pfn)
  3122. hole_pages = prev_end_pfn - range_start_pfn;
  3123. /* Find all holes for the zone within the node */
  3124. for (; i != -1; i = next_active_region_index_in_nid(i, nid)) {
  3125. /* No need to continue if prev_end_pfn is outside the zone */
  3126. if (prev_end_pfn >= range_end_pfn)
  3127. break;
  3128. /* Make sure the end of the zone is not within the hole */
  3129. start_pfn = min(early_node_map[i].start_pfn, range_end_pfn);
  3130. prev_end_pfn = max(prev_end_pfn, range_start_pfn);
  3131. /* Update the hole size cound and move on */
  3132. if (start_pfn > range_start_pfn) {
  3133. BUG_ON(prev_end_pfn > start_pfn);
  3134. hole_pages += start_pfn - prev_end_pfn;
  3135. }
  3136. prev_end_pfn = early_node_map[i].end_pfn;
  3137. }
  3138. /* Account for ranges past physical memory on this node */
  3139. if (range_end_pfn > prev_end_pfn)
  3140. hole_pages += range_end_pfn -
  3141. max(range_start_pfn, prev_end_pfn);
  3142. return hole_pages;
  3143. }
  3144. /**
  3145. * absent_pages_in_range - Return number of page frames in holes within a range
  3146. * @start_pfn: The start PFN to start searching for holes
  3147. * @end_pfn: The end PFN to stop searching for holes
  3148. *
  3149. * It returns the number of pages frames in memory holes within a range.
  3150. */
  3151. unsigned long __init absent_pages_in_range(unsigned long start_pfn,
  3152. unsigned long end_pfn)
  3153. {
  3154. return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
  3155. }
  3156. /* Return the number of page frames in holes in a zone on a node */
  3157. static unsigned long __meminit zone_absent_pages_in_node(int nid,
  3158. unsigned long zone_type,
  3159. unsigned long *ignored)
  3160. {
  3161. unsigned long node_start_pfn, node_end_pfn;
  3162. unsigned long zone_start_pfn, zone_end_pfn;
  3163. get_pfn_range_for_nid(nid, &node_start_pfn, &node_end_pfn);
  3164. zone_start_pfn = max(arch_zone_lowest_possible_pfn[zone_type],
  3165. node_start_pfn);
  3166. zone_end_pfn = min(arch_zone_highest_possible_pfn[zone_type],
  3167. node_end_pfn);
  3168. adjust_zone_range_for_zone_movable(nid, zone_type,
  3169. node_start_pfn, node_end_pfn,
  3170. &zone_start_pfn, &zone_end_pfn);
  3171. return __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
  3172. }
  3173. #else
  3174. static inline unsigned long __meminit zone_spanned_pages_in_node(int nid,
  3175. unsigned long zone_type,
  3176. unsigned long *zones_size)
  3177. {
  3178. return zones_size[zone_type];
  3179. }
  3180. static inline unsigned long __meminit zone_absent_pages_in_node(int nid,
  3181. unsigned long zone_type,
  3182. unsigned long *zholes_size)
  3183. {
  3184. if (!zholes_size)
  3185. return 0;
  3186. return zholes_size[zone_type];
  3187. }
  3188. #endif
  3189. static void __meminit calculate_node_totalpages(struct pglist_data *pgdat,
  3190. unsigned long *zones_size, unsigned long *zholes_size)
  3191. {
  3192. unsigned long realtotalpages, totalpages = 0;
  3193. enum zone_type i;
  3194. for (i = 0; i < MAX_NR_ZONES; i++)
  3195. totalpages += zone_spanned_pages_in_node(pgdat->node_id, i,
  3196. zones_size);
  3197. pgdat->node_spanned_pages = totalpages;
  3198. realtotalpages = totalpages;
  3199. for (i = 0; i < MAX_NR_ZONES; i++)
  3200. realtotalpages -=
  3201. zone_absent_pages_in_node(pgdat->node_id, i,
  3202. zholes_size);
  3203. pgdat->node_present_pages = realtotalpages;
  3204. printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id,
  3205. realtotalpages);
  3206. }
  3207. #ifndef CONFIG_SPARSEMEM
  3208. /*
  3209. * Calculate the size of the zone->blockflags rounded to an unsigned long
  3210. * Start by making sure zonesize is a multiple of pageblock_order by rounding
  3211. * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
  3212. * round what is now in bits to nearest long in bits, then return it in
  3213. * bytes.
  3214. */
  3215. static unsigned long __init usemap_size(unsigned long zonesize)
  3216. {
  3217. unsigned long usemapsize;
  3218. usemapsize = roundup(zonesize, pageblock_nr_pages);
  3219. usemapsize = usemapsize >> pageblock_order;
  3220. usemapsize *= NR_PAGEBLOCK_BITS;
  3221. usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
  3222. return usemapsize / 8;
  3223. }
  3224. static void __init setup_usemap(struct pglist_data *pgdat,
  3225. struct zone *zone, unsigned long zonesize)
  3226. {
  3227. unsigned long usemapsize = usemap_size(zonesize);
  3228. zone->pageblock_flags = NULL;
  3229. if (usemapsize)
  3230. zone->pageblock_flags = alloc_bootmem_node(pgdat, usemapsize);
  3231. }
  3232. #else
  3233. static void inline setup_usemap(struct pglist_data *pgdat,
  3234. struct zone *zone, unsigned long zonesize) {}
  3235. #endif /* CONFIG_SPARSEMEM */
  3236. #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
  3237. /* Return a sensible default order for the pageblock size. */
  3238. static inline int pageblock_default_order(void)
  3239. {
  3240. if (HPAGE_SHIFT > PAGE_SHIFT)
  3241. return HUGETLB_PAGE_ORDER;
  3242. return MAX_ORDER-1;
  3243. }
  3244. /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
  3245. static inline void __init set_pageblock_order(unsigned int order)
  3246. {
  3247. /* Check that pageblock_nr_pages has not already been setup */
  3248. if (pageblock_order)
  3249. return;
  3250. /*
  3251. * Assume the largest contiguous order of interest is a huge page.
  3252. * This value may be variable depending on boot parameters on IA64
  3253. */
  3254. pageblock_order = order;
  3255. }
  3256. #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
  3257. /*
  3258. * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
  3259. * and pageblock_default_order() are unused as pageblock_order is set
  3260. * at compile-time. See include/linux/pageblock-flags.h for the values of
  3261. * pageblock_order based on the kernel config
  3262. */
  3263. static inline int pageblock_default_order(unsigned int order)
  3264. {
  3265. return MAX_ORDER-1;
  3266. }
  3267. #define set_pageblock_order(x) do {} while (0)
  3268. #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
  3269. /*
  3270. * Set up the zone data structures:
  3271. * - mark all pages reserved
  3272. * - mark all memory queues empty
  3273. * - clear the memory bitmaps
  3274. */
  3275. static void __paginginit free_area_init_core(struct pglist_data *pgdat,
  3276. unsigned long *zones_size, unsigned long *zholes_size)
  3277. {
  3278. enum zone_type j;
  3279. int nid = pgdat->node_id;
  3280. unsigned long zone_start_pfn = pgdat->node_start_pfn;
  3281. int ret;
  3282. pgdat_resize_init(pgdat);
  3283. pgdat->nr_zones = 0;
  3284. init_waitqueue_head(&pgdat->kswapd_wait);
  3285. pgdat->kswapd_max_order = 0;
  3286. pgdat_page_cgroup_init(pgdat);
  3287. for (j = 0; j < MAX_NR_ZONES; j++) {
  3288. struct zone *zone = pgdat->node_zones + j;
  3289. unsigned long size, realsize, memmap_pages;
  3290. enum lru_list l;
  3291. size = zone_spanned_pages_in_node(nid, j, zones_size);
  3292. realsize = size - zone_absent_pages_in_node(nid, j,
  3293. zholes_size);
  3294. /*
  3295. * Adjust realsize so that it accounts for how much memory
  3296. * is used by this zone for memmap. This affects the watermark
  3297. * and per-cpu initialisations
  3298. */
  3299. memmap_pages =
  3300. PAGE_ALIGN(size * sizeof(struct page)) >> PAGE_SHIFT;
  3301. if (realsize >= memmap_pages) {
  3302. realsize -= memmap_pages;
  3303. if (memmap_pages)
  3304. printk(KERN_DEBUG
  3305. " %s zone: %lu pages used for memmap\n",
  3306. zone_names[j], memmap_pages);
  3307. } else
  3308. printk(KERN_WARNING
  3309. " %s zone: %lu pages exceeds realsize %lu\n",
  3310. zone_names[j], memmap_pages, realsize);
  3311. /* Account for reserved pages */
  3312. if (j == 0 && realsize > dma_reserve) {
  3313. realsize -= dma_reserve;
  3314. printk(KERN_DEBUG " %s zone: %lu pages reserved\n",
  3315. zone_names[0], dma_reserve);
  3316. }
  3317. if (!is_highmem_idx(j))
  3318. nr_kernel_pages += realsize;
  3319. nr_all_pages += realsize;
  3320. zone->spanned_pages = size;
  3321. zone->present_pages = realsize;
  3322. #ifdef CONFIG_NUMA
  3323. zone->node = nid;
  3324. zone->min_unmapped_pages = (realsize*sysctl_min_unmapped_ratio)
  3325. / 100;
  3326. zone->min_slab_pages = (realsize * sysctl_min_slab_ratio) / 100;
  3327. #endif
  3328. zone->name = zone_names[j];
  3329. spin_lock_init(&zone->lock);
  3330. spin_lock_init(&zone->lru_lock);
  3331. zone_seqlock_init(zone);
  3332. zone->zone_pgdat = pgdat;
  3333. zone->prev_priority = DEF_PRIORITY;
  3334. zone_pcp_init(zone);
  3335. for_each_lru(l) {
  3336. INIT_LIST_HEAD(&zone->lru[l].list);
  3337. zone->reclaim_stat.nr_saved_scan[l] = 0;
  3338. }
  3339. zone->reclaim_stat.recent_rotated[0] = 0;
  3340. zone->reclaim_stat.recent_rotated[1] = 0;
  3341. zone->reclaim_stat.recent_scanned[0] = 0;
  3342. zone->reclaim_stat.recent_scanned[1] = 0;
  3343. zap_zone_vm_stats(zone);
  3344. zone->flags = 0;
  3345. if (!size)
  3346. continue;
  3347. set_pageblock_order(pageblock_default_order());
  3348. setup_usemap(pgdat, zone, size);
  3349. ret = init_currently_empty_zone(zone, zone_start_pfn,
  3350. size, MEMMAP_EARLY);
  3351. BUG_ON(ret);
  3352. memmap_init(size, nid, j, zone_start_pfn);
  3353. zone_start_pfn += size;
  3354. }
  3355. }
  3356. static void __init_refok alloc_node_mem_map(struct pglist_data *pgdat)
  3357. {
  3358. /* Skip empty nodes */
  3359. if (!pgdat->node_spanned_pages)
  3360. return;
  3361. #ifdef CONFIG_FLAT_NODE_MEM_MAP
  3362. /* ia64 gets its own node_mem_map, before this, without bootmem */
  3363. if (!pgdat->node_mem_map) {
  3364. unsigned long size, start, end;
  3365. struct page *map;
  3366. /*
  3367. * The zone's endpoints aren't required to be MAX_ORDER
  3368. * aligned but the node_mem_map endpoints must be in order
  3369. * for the buddy allocator to function correctly.
  3370. */
  3371. start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
  3372. end = pgdat->node_start_pfn + pgdat->node_spanned_pages;
  3373. end = ALIGN(end, MAX_ORDER_NR_PAGES);
  3374. size = (end - start) * sizeof(struct page);
  3375. map = alloc_remap(pgdat->node_id, size);
  3376. if (!map)
  3377. map = alloc_bootmem_node(pgdat, size);
  3378. pgdat->node_mem_map = map + (pgdat->node_start_pfn - start);
  3379. }
  3380. #ifndef CONFIG_NEED_MULTIPLE_NODES
  3381. /*
  3382. * With no DISCONTIG, the global mem_map is just set as node 0's
  3383. */
  3384. if (pgdat == NODE_DATA(0)) {
  3385. mem_map = NODE_DATA(0)->node_mem_map;
  3386. #ifdef CONFIG_ARCH_POPULATES_NODE_MAP
  3387. if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
  3388. mem_map -= (pgdat->node_start_pfn - ARCH_PFN_OFFSET);
  3389. #endif /* CONFIG_ARCH_POPULATES_NODE_MAP */
  3390. }
  3391. #endif
  3392. #endif /* CONFIG_FLAT_NODE_MEM_MAP */
  3393. }
  3394. void __paginginit free_area_init_node(int nid, unsigned long *zones_size,
  3395. unsigned long node_start_pfn, unsigned long *zholes_size)
  3396. {
  3397. pg_data_t *pgdat = NODE_DATA(nid);
  3398. pgdat->node_id = nid;
  3399. pgdat->node_start_pfn = node_start_pfn;
  3400. calculate_node_totalpages(pgdat, zones_size, zholes_size);
  3401. alloc_node_mem_map(pgdat);
  3402. #ifdef CONFIG_FLAT_NODE_MEM_MAP
  3403. printk(KERN_DEBUG "free_area_init_node: node %d, pgdat %08lx, node_mem_map %08lx\n",
  3404. nid, (unsigned long)pgdat,
  3405. (unsigned long)pgdat->node_mem_map);
  3406. #endif
  3407. free_area_init_core(pgdat, zones_size, zholes_size);
  3408. }
  3409. #ifdef CONFIG_ARCH_POPULATES_NODE_MAP
  3410. #if MAX_NUMNODES > 1
  3411. /*
  3412. * Figure out the number of possible node ids.
  3413. */
  3414. static void __init setup_nr_node_ids(void)
  3415. {
  3416. unsigned int node;
  3417. unsigned int highest = 0;
  3418. for_each_node_mask(node, node_possible_map)
  3419. highest = node;
  3420. nr_node_ids = highest + 1;
  3421. }
  3422. #else
  3423. static inline void setup_nr_node_ids(void)
  3424. {
  3425. }
  3426. #endif
  3427. /**
  3428. * add_active_range - Register a range of PFNs backed by physical memory
  3429. * @nid: The node ID the range resides on
  3430. * @start_pfn: The start PFN of the available physical memory
  3431. * @end_pfn: The end PFN of the available physical memory
  3432. *
  3433. * These ranges are stored in an early_node_map[] and later used by
  3434. * free_area_init_nodes() to calculate zone sizes and holes. If the
  3435. * range spans a memory hole, it is up to the architecture to ensure
  3436. * the memory is not freed by the bootmem allocator. If possible
  3437. * the range being registered will be merged with existing ranges.
  3438. */
  3439. void __init add_active_range(unsigned int nid, unsigned long start_pfn,
  3440. unsigned long end_pfn)
  3441. {
  3442. int i;
  3443. mminit_dprintk(MMINIT_TRACE, "memory_register",
  3444. "Entering add_active_range(%d, %#lx, %#lx) "
  3445. "%d entries of %d used\n",
  3446. nid, start_pfn, end_pfn,
  3447. nr_nodemap_entries, MAX_ACTIVE_REGIONS);
  3448. mminit_validate_memmodel_limits(&start_pfn, &end_pfn);
  3449. /* Merge with existing active regions if possible */
  3450. for (i = 0; i < nr_nodemap_entries; i++) {
  3451. if (early_node_map[i].nid != nid)
  3452. continue;
  3453. /* Skip if an existing region covers this new one */
  3454. if (start_pfn >= early_node_map[i].start_pfn &&
  3455. end_pfn <= early_node_map[i].end_pfn)
  3456. return;
  3457. /* Merge forward if suitable */
  3458. if (start_pfn <= early_node_map[i].end_pfn &&
  3459. end_pfn > early_node_map[i].end_pfn) {
  3460. early_node_map[i].end_pfn = end_pfn;
  3461. return;
  3462. }
  3463. /* Merge backward if suitable */
  3464. if (start_pfn < early_node_map[i].start_pfn &&
  3465. end_pfn >= early_node_map[i].start_pfn) {
  3466. early_node_map[i].start_pfn = start_pfn;
  3467. return;
  3468. }
  3469. }
  3470. /* Check that early_node_map is large enough */
  3471. if (i >= MAX_ACTIVE_REGIONS) {
  3472. printk(KERN_CRIT "More than %d memory regions, truncating\n",
  3473. MAX_ACTIVE_REGIONS);
  3474. return;
  3475. }
  3476. early_node_map[i].nid = nid;
  3477. early_node_map[i].start_pfn = start_pfn;
  3478. early_node_map[i].end_pfn = end_pfn;
  3479. nr_nodemap_entries = i + 1;
  3480. }
  3481. /**
  3482. * remove_active_range - Shrink an existing registered range of PFNs
  3483. * @nid: The node id the range is on that should be shrunk
  3484. * @start_pfn: The new PFN of the range
  3485. * @end_pfn: The new PFN of the range
  3486. *
  3487. * i386 with NUMA use alloc_remap() to store a node_mem_map on a local node.
  3488. * The map is kept near the end physical page range that has already been
  3489. * registered. This function allows an arch to shrink an existing registered
  3490. * range.
  3491. */
  3492. void __init remove_active_range(unsigned int nid, unsigned long start_pfn,
  3493. unsigned long end_pfn)
  3494. {
  3495. int i, j;
  3496. int removed = 0;
  3497. printk(KERN_DEBUG "remove_active_range (%d, %lu, %lu)\n",
  3498. nid, start_pfn, end_pfn);
  3499. /* Find the old active region end and shrink */
  3500. for_each_active_range_index_in_nid(i, nid) {
  3501. if (early_node_map[i].start_pfn >= start_pfn &&
  3502. early_node_map[i].end_pfn <= end_pfn) {
  3503. /* clear it */
  3504. early_node_map[i].start_pfn = 0;
  3505. early_node_map[i].end_pfn = 0;
  3506. removed = 1;
  3507. continue;
  3508. }
  3509. if (early_node_map[i].start_pfn < start_pfn &&
  3510. early_node_map[i].end_pfn > start_pfn) {
  3511. unsigned long temp_end_pfn = early_node_map[i].end_pfn;
  3512. early_node_map[i].end_pfn = start_pfn;
  3513. if (temp_end_pfn > end_pfn)
  3514. add_active_range(nid, end_pfn, temp_end_pfn);
  3515. continue;
  3516. }
  3517. if (early_node_map[i].start_pfn >= start_pfn &&
  3518. early_node_map[i].end_pfn > end_pfn &&
  3519. early_node_map[i].start_pfn < end_pfn) {
  3520. early_node_map[i].start_pfn = end_pfn;
  3521. continue;
  3522. }
  3523. }
  3524. if (!removed)
  3525. return;
  3526. /* remove the blank ones */
  3527. for (i = nr_nodemap_entries - 1; i > 0; i--) {
  3528. if (early_node_map[i].nid != nid)
  3529. continue;
  3530. if (early_node_map[i].end_pfn)
  3531. continue;
  3532. /* we found it, get rid of it */
  3533. for (j = i; j < nr_nodemap_entries - 1; j++)
  3534. memcpy(&early_node_map[j], &early_node_map[j+1],
  3535. sizeof(early_node_map[j]));
  3536. j = nr_nodemap_entries - 1;
  3537. memset(&early_node_map[j], 0, sizeof(early_node_map[j]));
  3538. nr_nodemap_entries--;
  3539. }
  3540. }
  3541. /**
  3542. * remove_all_active_ranges - Remove all currently registered regions
  3543. *
  3544. * During discovery, it may be found that a table like SRAT is invalid
  3545. * and an alternative discovery method must be used. This function removes
  3546. * all currently registered regions.
  3547. */
  3548. void __init remove_all_active_ranges(void)
  3549. {
  3550. memset(early_node_map, 0, sizeof(early_node_map));
  3551. nr_nodemap_entries = 0;
  3552. }
  3553. /* Compare two active node_active_regions */
  3554. static int __init cmp_node_active_region(const void *a, const void *b)
  3555. {
  3556. struct node_active_region *arange = (struct node_active_region *)a;
  3557. struct node_active_region *brange = (struct node_active_region *)b;
  3558. /* Done this way to avoid overflows */
  3559. if (arange->start_pfn > brange->start_pfn)
  3560. return 1;
  3561. if (arange->start_pfn < brange->start_pfn)
  3562. return -1;
  3563. return 0;
  3564. }
  3565. /* sort the node_map by start_pfn */
  3566. void __init sort_node_map(void)
  3567. {
  3568. sort(early_node_map, (size_t)nr_nodemap_entries,
  3569. sizeof(struct node_active_region),
  3570. cmp_node_active_region, NULL);
  3571. }
  3572. /* Find the lowest pfn for a node */
  3573. static unsigned long __init find_min_pfn_for_node(int nid)
  3574. {
  3575. int i;
  3576. unsigned long min_pfn = ULONG_MAX;
  3577. /* Assuming a sorted map, the first range found has the starting pfn */
  3578. for_each_active_range_index_in_nid(i, nid)
  3579. min_pfn = min(min_pfn, early_node_map[i].start_pfn);
  3580. if (min_pfn == ULONG_MAX) {
  3581. printk(KERN_WARNING
  3582. "Could not find start_pfn for node %d\n", nid);
  3583. return 0;
  3584. }
  3585. return min_pfn;
  3586. }
  3587. /**
  3588. * find_min_pfn_with_active_regions - Find the minimum PFN registered
  3589. *
  3590. * It returns the minimum PFN based on information provided via
  3591. * add_active_range().
  3592. */
  3593. unsigned long __init find_min_pfn_with_active_regions(void)
  3594. {
  3595. return find_min_pfn_for_node(MAX_NUMNODES);
  3596. }
  3597. /*
  3598. * early_calculate_totalpages()
  3599. * Sum pages in active regions for movable zone.
  3600. * Populate N_HIGH_MEMORY for calculating usable_nodes.
  3601. */
  3602. static unsigned long __init early_calculate_totalpages(void)
  3603. {
  3604. int i;
  3605. unsigned long totalpages = 0;
  3606. for (i = 0; i < nr_nodemap_entries; i++) {
  3607. unsigned long pages = early_node_map[i].end_pfn -
  3608. early_node_map[i].start_pfn;
  3609. totalpages += pages;
  3610. if (pages)
  3611. node_set_state(early_node_map[i].nid, N_HIGH_MEMORY);
  3612. }
  3613. return totalpages;
  3614. }
  3615. /*
  3616. * Find the PFN the Movable zone begins in each node. Kernel memory
  3617. * is spread evenly between nodes as long as the nodes have enough
  3618. * memory. When they don't, some nodes will have more kernelcore than
  3619. * others
  3620. */
  3621. static void __init find_zone_movable_pfns_for_nodes(unsigned long *movable_pfn)
  3622. {
  3623. int i, nid;
  3624. unsigned long usable_startpfn;
  3625. unsigned long kernelcore_node, kernelcore_remaining;
  3626. /* save the state before borrow the nodemask */
  3627. nodemask_t saved_node_state = node_states[N_HIGH_MEMORY];
  3628. unsigned long totalpages = early_calculate_totalpages();
  3629. int usable_nodes = nodes_weight(node_states[N_HIGH_MEMORY]);
  3630. /*
  3631. * If movablecore was specified, calculate what size of
  3632. * kernelcore that corresponds so that memory usable for
  3633. * any allocation type is evenly spread. If both kernelcore
  3634. * and movablecore are specified, then the value of kernelcore
  3635. * will be used for required_kernelcore if it's greater than
  3636. * what movablecore would have allowed.
  3637. */
  3638. if (required_movablecore) {
  3639. unsigned long corepages;
  3640. /*
  3641. * Round-up so that ZONE_MOVABLE is at least as large as what
  3642. * was requested by the user
  3643. */
  3644. required_movablecore =
  3645. roundup(required_movablecore, MAX_ORDER_NR_PAGES);
  3646. corepages = totalpages - required_movablecore;
  3647. required_kernelcore = max(required_kernelcore, corepages);
  3648. }
  3649. /* If kernelcore was not specified, there is no ZONE_MOVABLE */
  3650. if (!required_kernelcore)
  3651. goto out;
  3652. /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
  3653. find_usable_zone_for_movable();
  3654. usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
  3655. restart:
  3656. /* Spread kernelcore memory as evenly as possible throughout nodes */
  3657. kernelcore_node = required_kernelcore / usable_nodes;
  3658. for_each_node_state(nid, N_HIGH_MEMORY) {
  3659. /*
  3660. * Recalculate kernelcore_node if the division per node
  3661. * now exceeds what is necessary to satisfy the requested
  3662. * amount of memory for the kernel
  3663. */
  3664. if (required_kernelcore < kernelcore_node)
  3665. kernelcore_node = required_kernelcore / usable_nodes;
  3666. /*
  3667. * As the map is walked, we track how much memory is usable
  3668. * by the kernel using kernelcore_remaining. When it is
  3669. * 0, the rest of the node is usable by ZONE_MOVABLE
  3670. */
  3671. kernelcore_remaining = kernelcore_node;
  3672. /* Go through each range of PFNs within this node */
  3673. for_each_active_range_index_in_nid(i, nid) {
  3674. unsigned long start_pfn, end_pfn;
  3675. unsigned long size_pages;
  3676. start_pfn = max(early_node_map[i].start_pfn,
  3677. zone_movable_pfn[nid]);
  3678. end_pfn = early_node_map[i].end_pfn;
  3679. if (start_pfn >= end_pfn)
  3680. continue;
  3681. /* Account for what is only usable for kernelcore */
  3682. if (start_pfn < usable_startpfn) {
  3683. unsigned long kernel_pages;
  3684. kernel_pages = min(end_pfn, usable_startpfn)
  3685. - start_pfn;
  3686. kernelcore_remaining -= min(kernel_pages,
  3687. kernelcore_remaining);
  3688. required_kernelcore -= min(kernel_pages,
  3689. required_kernelcore);
  3690. /* Continue if range is now fully accounted */
  3691. if (end_pfn <= usable_startpfn) {
  3692. /*
  3693. * Push zone_movable_pfn to the end so
  3694. * that if we have to rebalance
  3695. * kernelcore across nodes, we will
  3696. * not double account here
  3697. */
  3698. zone_movable_pfn[nid] = end_pfn;
  3699. continue;
  3700. }
  3701. start_pfn = usable_startpfn;
  3702. }
  3703. /*
  3704. * The usable PFN range for ZONE_MOVABLE is from
  3705. * start_pfn->end_pfn. Calculate size_pages as the
  3706. * number of pages used as kernelcore
  3707. */
  3708. size_pages = end_pfn - start_pfn;
  3709. if (size_pages > kernelcore_remaining)
  3710. size_pages = kernelcore_remaining;
  3711. zone_movable_pfn[nid] = start_pfn + size_pages;
  3712. /*
  3713. * Some kernelcore has been met, update counts and
  3714. * break if the kernelcore for this node has been
  3715. * satisified
  3716. */
  3717. required_kernelcore -= min(required_kernelcore,
  3718. size_pages);
  3719. kernelcore_remaining -= size_pages;
  3720. if (!kernelcore_remaining)
  3721. break;
  3722. }
  3723. }
  3724. /*
  3725. * If there is still required_kernelcore, we do another pass with one
  3726. * less node in the count. This will push zone_movable_pfn[nid] further
  3727. * along on the nodes that still have memory until kernelcore is
  3728. * satisified
  3729. */
  3730. usable_nodes--;
  3731. if (usable_nodes && required_kernelcore > usable_nodes)
  3732. goto restart;
  3733. /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
  3734. for (nid = 0; nid < MAX_NUMNODES; nid++)
  3735. zone_movable_pfn[nid] =
  3736. roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
  3737. out:
  3738. /* restore the node_state */
  3739. node_states[N_HIGH_MEMORY] = saved_node_state;
  3740. }
  3741. /* Any regular memory on that node ? */
  3742. static void check_for_regular_memory(pg_data_t *pgdat)
  3743. {
  3744. #ifdef CONFIG_HIGHMEM
  3745. enum zone_type zone_type;
  3746. for (zone_type = 0; zone_type <= ZONE_NORMAL; zone_type++) {
  3747. struct zone *zone = &pgdat->node_zones[zone_type];
  3748. if (zone->present_pages)
  3749. node_set_state(zone_to_nid(zone), N_NORMAL_MEMORY);
  3750. }
  3751. #endif
  3752. }
  3753. /**
  3754. * free_area_init_nodes - Initialise all pg_data_t and zone data
  3755. * @max_zone_pfn: an array of max PFNs for each zone
  3756. *
  3757. * This will call free_area_init_node() for each active node in the system.
  3758. * Using the page ranges provided by add_active_range(), the size of each
  3759. * zone in each node and their holes is calculated. If the maximum PFN
  3760. * between two adjacent zones match, it is assumed that the zone is empty.
  3761. * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
  3762. * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
  3763. * starts where the previous one ended. For example, ZONE_DMA32 starts
  3764. * at arch_max_dma_pfn.
  3765. */
  3766. void __init free_area_init_nodes(unsigned long *max_zone_pfn)
  3767. {
  3768. unsigned long nid;
  3769. int i;
  3770. /* Sort early_node_map as initialisation assumes it is sorted */
  3771. sort_node_map();
  3772. /* Record where the zone boundaries are */
  3773. memset(arch_zone_lowest_possible_pfn, 0,
  3774. sizeof(arch_zone_lowest_possible_pfn));
  3775. memset(arch_zone_highest_possible_pfn, 0,
  3776. sizeof(arch_zone_highest_possible_pfn));
  3777. arch_zone_lowest_possible_pfn[0] = find_min_pfn_with_active_regions();
  3778. arch_zone_highest_possible_pfn[0] = max_zone_pfn[0];
  3779. for (i = 1; i < MAX_NR_ZONES; i++) {
  3780. if (i == ZONE_MOVABLE)
  3781. continue;
  3782. arch_zone_lowest_possible_pfn[i] =
  3783. arch_zone_highest_possible_pfn[i-1];
  3784. arch_zone_highest_possible_pfn[i] =
  3785. max(max_zone_pfn[i], arch_zone_lowest_possible_pfn[i]);
  3786. }
  3787. arch_zone_lowest_possible_pfn[ZONE_MOVABLE] = 0;
  3788. arch_zone_highest_possible_pfn[ZONE_MOVABLE] = 0;
  3789. /* Find the PFNs that ZONE_MOVABLE begins at in each node */
  3790. memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
  3791. find_zone_movable_pfns_for_nodes(zone_movable_pfn);
  3792. /* Print out the zone ranges */
  3793. printk("Zone PFN ranges:\n");
  3794. for (i = 0; i < MAX_NR_ZONES; i++) {
  3795. if (i == ZONE_MOVABLE)
  3796. continue;
  3797. printk(" %-8s ", zone_names[i]);
  3798. if (arch_zone_lowest_possible_pfn[i] ==
  3799. arch_zone_highest_possible_pfn[i])
  3800. printk("empty\n");
  3801. else
  3802. printk("%0#10lx -> %0#10lx\n",
  3803. arch_zone_lowest_possible_pfn[i],
  3804. arch_zone_highest_possible_pfn[i]);
  3805. }
  3806. /* Print out the PFNs ZONE_MOVABLE begins at in each node */
  3807. printk("Movable zone start PFN for each node\n");
  3808. for (i = 0; i < MAX_NUMNODES; i++) {
  3809. if (zone_movable_pfn[i])
  3810. printk(" Node %d: %lu\n", i, zone_movable_pfn[i]);
  3811. }
  3812. /* Print out the early_node_map[] */
  3813. printk("early_node_map[%d] active PFN ranges\n", nr_nodemap_entries);
  3814. for (i = 0; i < nr_nodemap_entries; i++)
  3815. printk(" %3d: %0#10lx -> %0#10lx\n", early_node_map[i].nid,
  3816. early_node_map[i].start_pfn,
  3817. early_node_map[i].end_pfn);
  3818. /* Initialise every node */
  3819. mminit_verify_pageflags_layout();
  3820. setup_nr_node_ids();
  3821. for_each_online_node(nid) {
  3822. pg_data_t *pgdat = NODE_DATA(nid);
  3823. free_area_init_node(nid, NULL,
  3824. find_min_pfn_for_node(nid), NULL);
  3825. /* Any memory on that node */
  3826. if (pgdat->node_present_pages)
  3827. node_set_state(nid, N_HIGH_MEMORY);
  3828. check_for_regular_memory(pgdat);
  3829. }
  3830. }
  3831. static int __init cmdline_parse_core(char *p, unsigned long *core)
  3832. {
  3833. unsigned long long coremem;
  3834. if (!p)
  3835. return -EINVAL;
  3836. coremem = memparse(p, &p);
  3837. *core = coremem >> PAGE_SHIFT;
  3838. /* Paranoid check that UL is enough for the coremem value */
  3839. WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
  3840. return 0;
  3841. }
  3842. /*
  3843. * kernelcore=size sets the amount of memory for use for allocations that
  3844. * cannot be reclaimed or migrated.
  3845. */
  3846. static int __init cmdline_parse_kernelcore(char *p)
  3847. {
  3848. return cmdline_parse_core(p, &required_kernelcore);
  3849. }
  3850. /*
  3851. * movablecore=size sets the amount of memory for use for allocations that
  3852. * can be reclaimed or migrated.
  3853. */
  3854. static int __init cmdline_parse_movablecore(char *p)
  3855. {
  3856. return cmdline_parse_core(p, &required_movablecore);
  3857. }
  3858. early_param("kernelcore", cmdline_parse_kernelcore);
  3859. early_param("movablecore", cmdline_parse_movablecore);
  3860. #endif /* CONFIG_ARCH_POPULATES_NODE_MAP */
  3861. /**
  3862. * set_dma_reserve - set the specified number of pages reserved in the first zone
  3863. * @new_dma_reserve: The number of pages to mark reserved
  3864. *
  3865. * The per-cpu batchsize and zone watermarks are determined by present_pages.
  3866. * In the DMA zone, a significant percentage may be consumed by kernel image
  3867. * and other unfreeable allocations which can skew the watermarks badly. This
  3868. * function may optionally be used to account for unfreeable pages in the
  3869. * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
  3870. * smaller per-cpu batchsize.
  3871. */
  3872. void __init set_dma_reserve(unsigned long new_dma_reserve)
  3873. {
  3874. dma_reserve = new_dma_reserve;
  3875. }
  3876. #ifndef CONFIG_NEED_MULTIPLE_NODES
  3877. struct pglist_data __refdata contig_page_data = {
  3878. #ifndef CONFIG_NO_BOOTMEM
  3879. .bdata = &bootmem_node_data[0]
  3880. #endif
  3881. };
  3882. EXPORT_SYMBOL(contig_page_data);
  3883. #endif
  3884. void __init free_area_init(unsigned long *zones_size)
  3885. {
  3886. free_area_init_node(0, zones_size,
  3887. __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL);
  3888. }
  3889. static int page_alloc_cpu_notify(struct notifier_block *self,
  3890. unsigned long action, void *hcpu)
  3891. {
  3892. int cpu = (unsigned long)hcpu;
  3893. if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
  3894. drain_pages(cpu);
  3895. /*
  3896. * Spill the event counters of the dead processor
  3897. * into the current processors event counters.
  3898. * This artificially elevates the count of the current
  3899. * processor.
  3900. */
  3901. vm_events_fold_cpu(cpu);
  3902. /*
  3903. * Zero the differential counters of the dead processor
  3904. * so that the vm statistics are consistent.
  3905. *
  3906. * This is only okay since the processor is dead and cannot
  3907. * race with what we are doing.
  3908. */
  3909. refresh_cpu_vm_stats(cpu);
  3910. }
  3911. return NOTIFY_OK;
  3912. }
  3913. void __init page_alloc_init(void)
  3914. {
  3915. hotcpu_notifier(page_alloc_cpu_notify, 0);
  3916. }
  3917. /*
  3918. * calculate_totalreserve_pages - called when sysctl_lower_zone_reserve_ratio
  3919. * or min_free_kbytes changes.
  3920. */
  3921. static void calculate_totalreserve_pages(void)
  3922. {
  3923. struct pglist_data *pgdat;
  3924. unsigned long reserve_pages = 0;
  3925. enum zone_type i, j;
  3926. for_each_online_pgdat(pgdat) {
  3927. for (i = 0; i < MAX_NR_ZONES; i++) {
  3928. struct zone *zone = pgdat->node_zones + i;
  3929. unsigned long max = 0;
  3930. /* Find valid and maximum lowmem_reserve in the zone */
  3931. for (j = i; j < MAX_NR_ZONES; j++) {
  3932. if (zone->lowmem_reserve[j] > max)
  3933. max = zone->lowmem_reserve[j];
  3934. }
  3935. /* we treat the high watermark as reserved pages. */
  3936. max += high_wmark_pages(zone);
  3937. if (max > zone->present_pages)
  3938. max = zone->present_pages;
  3939. reserve_pages += max;
  3940. }
  3941. }
  3942. totalreserve_pages = reserve_pages;
  3943. }
  3944. /*
  3945. * setup_per_zone_lowmem_reserve - called whenever
  3946. * sysctl_lower_zone_reserve_ratio changes. Ensures that each zone
  3947. * has a correct pages reserved value, so an adequate number of
  3948. * pages are left in the zone after a successful __alloc_pages().
  3949. */
  3950. static void setup_per_zone_lowmem_reserve(void)
  3951. {
  3952. struct pglist_data *pgdat;
  3953. enum zone_type j, idx;
  3954. for_each_online_pgdat(pgdat) {
  3955. for (j = 0; j < MAX_NR_ZONES; j++) {
  3956. struct zone *zone = pgdat->node_zones + j;
  3957. unsigned long present_pages = zone->present_pages;
  3958. zone->lowmem_reserve[j] = 0;
  3959. idx = j;
  3960. while (idx) {
  3961. struct zone *lower_zone;
  3962. idx--;
  3963. if (sysctl_lowmem_reserve_ratio[idx] < 1)
  3964. sysctl_lowmem_reserve_ratio[idx] = 1;
  3965. lower_zone = pgdat->node_zones + idx;
  3966. lower_zone->lowmem_reserve[j] = present_pages /
  3967. sysctl_lowmem_reserve_ratio[idx];
  3968. present_pages += lower_zone->present_pages;
  3969. }
  3970. }
  3971. }
  3972. /* update totalreserve_pages */
  3973. calculate_totalreserve_pages();
  3974. }
  3975. /**
  3976. * setup_per_zone_wmarks - called when min_free_kbytes changes
  3977. * or when memory is hot-{added|removed}
  3978. *
  3979. * Ensures that the watermark[min,low,high] values for each zone are set
  3980. * correctly with respect to min_free_kbytes.
  3981. */
  3982. void setup_per_zone_wmarks(void)
  3983. {
  3984. unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
  3985. unsigned long lowmem_pages = 0;
  3986. struct zone *zone;
  3987. unsigned long flags;
  3988. /* Calculate total number of !ZONE_HIGHMEM pages */
  3989. for_each_zone(zone) {
  3990. if (!is_highmem(zone))
  3991. lowmem_pages += zone->present_pages;
  3992. }
  3993. for_each_zone(zone) {
  3994. u64 tmp;
  3995. spin_lock_irqsave(&zone->lock, flags);
  3996. tmp = (u64)pages_min * zone->present_pages;
  3997. do_div(tmp, lowmem_pages);
  3998. if (is_highmem(zone)) {
  3999. /*
  4000. * __GFP_HIGH and PF_MEMALLOC allocations usually don't
  4001. * need highmem pages, so cap pages_min to a small
  4002. * value here.
  4003. *
  4004. * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
  4005. * deltas controls asynch page reclaim, and so should
  4006. * not be capped for highmem.
  4007. */
  4008. int min_pages;
  4009. min_pages = zone->present_pages / 1024;
  4010. if (min_pages < SWAP_CLUSTER_MAX)
  4011. min_pages = SWAP_CLUSTER_MAX;
  4012. if (min_pages > 128)
  4013. min_pages = 128;
  4014. zone->watermark[WMARK_MIN] = min_pages;
  4015. } else {
  4016. /*
  4017. * If it's a lowmem zone, reserve a number of pages
  4018. * proportionate to the zone's size.
  4019. */
  4020. zone->watermark[WMARK_MIN] = tmp;
  4021. }
  4022. zone->watermark[WMARK_LOW] = min_wmark_pages(zone) + (tmp >> 2);
  4023. zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + (tmp >> 1);
  4024. setup_zone_migrate_reserve(zone);
  4025. spin_unlock_irqrestore(&zone->lock, flags);
  4026. }
  4027. /* update totalreserve_pages */
  4028. calculate_totalreserve_pages();
  4029. }
  4030. /*
  4031. * The inactive anon list should be small enough that the VM never has to
  4032. * do too much work, but large enough that each inactive page has a chance
  4033. * to be referenced again before it is swapped out.
  4034. *
  4035. * The inactive_anon ratio is the target ratio of ACTIVE_ANON to
  4036. * INACTIVE_ANON pages on this zone's LRU, maintained by the
  4037. * pageout code. A zone->inactive_ratio of 3 means 3:1 or 25% of
  4038. * the anonymous pages are kept on the inactive list.
  4039. *
  4040. * total target max
  4041. * memory ratio inactive anon
  4042. * -------------------------------------
  4043. * 10MB 1 5MB
  4044. * 100MB 1 50MB
  4045. * 1GB 3 250MB
  4046. * 10GB 10 0.9GB
  4047. * 100GB 31 3GB
  4048. * 1TB 101 10GB
  4049. * 10TB 320 32GB
  4050. */
  4051. void calculate_zone_inactive_ratio(struct zone *zone)
  4052. {
  4053. unsigned int gb, ratio;
  4054. /* Zone size in gigabytes */
  4055. gb = zone->present_pages >> (30 - PAGE_SHIFT);
  4056. if (gb)
  4057. ratio = int_sqrt(10 * gb);
  4058. else
  4059. ratio = 1;
  4060. zone->inactive_ratio = ratio;
  4061. }
  4062. static void __init setup_per_zone_inactive_ratio(void)
  4063. {
  4064. struct zone *zone;
  4065. for_each_zone(zone)
  4066. calculate_zone_inactive_ratio(zone);
  4067. }
  4068. /*
  4069. * Initialise min_free_kbytes.
  4070. *
  4071. * For small machines we want it small (128k min). For large machines
  4072. * we want it large (64MB max). But it is not linear, because network
  4073. * bandwidth does not increase linearly with machine size. We use
  4074. *
  4075. * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy:
  4076. * min_free_kbytes = sqrt(lowmem_kbytes * 16)
  4077. *
  4078. * which yields
  4079. *
  4080. * 16MB: 512k
  4081. * 32MB: 724k
  4082. * 64MB: 1024k
  4083. * 128MB: 1448k
  4084. * 256MB: 2048k
  4085. * 512MB: 2896k
  4086. * 1024MB: 4096k
  4087. * 2048MB: 5792k
  4088. * 4096MB: 8192k
  4089. * 8192MB: 11584k
  4090. * 16384MB: 16384k
  4091. */
  4092. static int __init init_per_zone_wmark_min(void)
  4093. {
  4094. unsigned long lowmem_kbytes;
  4095. lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
  4096. min_free_kbytes = int_sqrt(lowmem_kbytes * 16);
  4097. if (min_free_kbytes < 128)
  4098. min_free_kbytes = 128;
  4099. if (min_free_kbytes > 65536)
  4100. min_free_kbytes = 65536;
  4101. setup_per_zone_wmarks();
  4102. setup_per_zone_lowmem_reserve();
  4103. setup_per_zone_inactive_ratio();
  4104. return 0;
  4105. }
  4106. module_init(init_per_zone_wmark_min)
  4107. /*
  4108. * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so
  4109. * that we can call two helper functions whenever min_free_kbytes
  4110. * changes.
  4111. */
  4112. int min_free_kbytes_sysctl_handler(ctl_table *table, int write,
  4113. void __user *buffer, size_t *length, loff_t *ppos)
  4114. {
  4115. proc_dointvec(table, write, buffer, length, ppos);
  4116. if (write)
  4117. setup_per_zone_wmarks();
  4118. return 0;
  4119. }
  4120. #ifdef CONFIG_NUMA
  4121. int sysctl_min_unmapped_ratio_sysctl_handler(ctl_table *table, int write,
  4122. void __user *buffer, size_t *length, loff_t *ppos)
  4123. {
  4124. struct zone *zone;
  4125. int rc;
  4126. rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
  4127. if (rc)
  4128. return rc;
  4129. for_each_zone(zone)
  4130. zone->min_unmapped_pages = (zone->present_pages *
  4131. sysctl_min_unmapped_ratio) / 100;
  4132. return 0;
  4133. }
  4134. int sysctl_min_slab_ratio_sysctl_handler(ctl_table *table, int write,
  4135. void __user *buffer, size_t *length, loff_t *ppos)
  4136. {
  4137. struct zone *zone;
  4138. int rc;
  4139. rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
  4140. if (rc)
  4141. return rc;
  4142. for_each_zone(zone)
  4143. zone->min_slab_pages = (zone->present_pages *
  4144. sysctl_min_slab_ratio) / 100;
  4145. return 0;
  4146. }
  4147. #endif
  4148. /*
  4149. * lowmem_reserve_ratio_sysctl_handler - just a wrapper around
  4150. * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve()
  4151. * whenever sysctl_lowmem_reserve_ratio changes.
  4152. *
  4153. * The reserve ratio obviously has absolutely no relation with the
  4154. * minimum watermarks. The lowmem reserve ratio can only make sense
  4155. * if in function of the boot time zone sizes.
  4156. */
  4157. int lowmem_reserve_ratio_sysctl_handler(ctl_table *table, int write,
  4158. void __user *buffer, size_t *length, loff_t *ppos)
  4159. {
  4160. proc_dointvec_minmax(table, write, buffer, length, ppos);
  4161. setup_per_zone_lowmem_reserve();
  4162. return 0;
  4163. }
  4164. /*
  4165. * percpu_pagelist_fraction - changes the pcp->high for each zone on each
  4166. * cpu. It is the fraction of total pages in each zone that a hot per cpu pagelist
  4167. * can have before it gets flushed back to buddy allocator.
  4168. */
  4169. int percpu_pagelist_fraction_sysctl_handler(ctl_table *table, int write,
  4170. void __user *buffer, size_t *length, loff_t *ppos)
  4171. {
  4172. struct zone *zone;
  4173. unsigned int cpu;
  4174. int ret;
  4175. ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
  4176. if (!write || (ret == -EINVAL))
  4177. return ret;
  4178. for_each_populated_zone(zone) {
  4179. for_each_possible_cpu(cpu) {
  4180. unsigned long high;
  4181. high = zone->present_pages / percpu_pagelist_fraction;
  4182. setup_pagelist_highmark(
  4183. per_cpu_ptr(zone->pageset, cpu), high);
  4184. }
  4185. }
  4186. return 0;
  4187. }
  4188. int hashdist = HASHDIST_DEFAULT;
  4189. #ifdef CONFIG_NUMA
  4190. static int __init set_hashdist(char *str)
  4191. {
  4192. if (!str)
  4193. return 0;
  4194. hashdist = simple_strtoul(str, &str, 0);
  4195. return 1;
  4196. }
  4197. __setup("hashdist=", set_hashdist);
  4198. #endif
  4199. /*
  4200. * allocate a large system hash table from bootmem
  4201. * - it is assumed that the hash table must contain an exact power-of-2
  4202. * quantity of entries
  4203. * - limit is the number of hash buckets, not the total allocation size
  4204. */
  4205. void *__init alloc_large_system_hash(const char *tablename,
  4206. unsigned long bucketsize,
  4207. unsigned long numentries,
  4208. int scale,
  4209. int flags,
  4210. unsigned int *_hash_shift,
  4211. unsigned int *_hash_mask,
  4212. unsigned long limit)
  4213. {
  4214. unsigned long long max = limit;
  4215. unsigned long log2qty, size;
  4216. void *table = NULL;
  4217. /* allow the kernel cmdline to have a say */
  4218. if (!numentries) {
  4219. /* round applicable memory size up to nearest megabyte */
  4220. numentries = nr_kernel_pages;
  4221. numentries += (1UL << (20 - PAGE_SHIFT)) - 1;
  4222. numentries >>= 20 - PAGE_SHIFT;
  4223. numentries <<= 20 - PAGE_SHIFT;
  4224. /* limit to 1 bucket per 2^scale bytes of low memory */
  4225. if (scale > PAGE_SHIFT)
  4226. numentries >>= (scale - PAGE_SHIFT);
  4227. else
  4228. numentries <<= (PAGE_SHIFT - scale);
  4229. /* Make sure we've got at least a 0-order allocation.. */
  4230. if (unlikely(flags & HASH_SMALL)) {
  4231. /* Makes no sense without HASH_EARLY */
  4232. WARN_ON(!(flags & HASH_EARLY));
  4233. if (!(numentries >> *_hash_shift)) {
  4234. numentries = 1UL << *_hash_shift;
  4235. BUG_ON(!numentries);
  4236. }
  4237. } else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
  4238. numentries = PAGE_SIZE / bucketsize;
  4239. }
  4240. numentries = roundup_pow_of_two(numentries);
  4241. /* limit allocation size to 1/16 total memory by default */
  4242. if (max == 0) {
  4243. max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
  4244. do_div(max, bucketsize);
  4245. }
  4246. if (numentries > max)
  4247. numentries = max;
  4248. log2qty = ilog2(numentries);
  4249. do {
  4250. size = bucketsize << log2qty;
  4251. if (flags & HASH_EARLY)
  4252. table = alloc_bootmem_nopanic(size);
  4253. else if (hashdist)
  4254. table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL);
  4255. else {
  4256. /*
  4257. * If bucketsize is not a power-of-two, we may free
  4258. * some pages at the end of hash table which
  4259. * alloc_pages_exact() automatically does
  4260. */
  4261. if (get_order(size) < MAX_ORDER) {
  4262. table = alloc_pages_exact(size, GFP_ATOMIC);
  4263. kmemleak_alloc(table, size, 1, GFP_ATOMIC);
  4264. }
  4265. }
  4266. } while (!table && size > PAGE_SIZE && --log2qty);
  4267. if (!table)
  4268. panic("Failed to allocate %s hash table\n", tablename);
  4269. printk(KERN_INFO "%s hash table entries: %d (order: %d, %lu bytes)\n",
  4270. tablename,
  4271. (1U << log2qty),
  4272. ilog2(size) - PAGE_SHIFT,
  4273. size);
  4274. if (_hash_shift)
  4275. *_hash_shift = log2qty;
  4276. if (_hash_mask)
  4277. *_hash_mask = (1 << log2qty) - 1;
  4278. return table;
  4279. }
  4280. /* Return a pointer to the bitmap storing bits affecting a block of pages */
  4281. static inline unsigned long *get_pageblock_bitmap(struct zone *zone,
  4282. unsigned long pfn)
  4283. {
  4284. #ifdef CONFIG_SPARSEMEM
  4285. return __pfn_to_section(pfn)->pageblock_flags;
  4286. #else
  4287. return zone->pageblock_flags;
  4288. #endif /* CONFIG_SPARSEMEM */
  4289. }
  4290. static inline int pfn_to_bitidx(struct zone *zone, unsigned long pfn)
  4291. {
  4292. #ifdef CONFIG_SPARSEMEM
  4293. pfn &= (PAGES_PER_SECTION-1);
  4294. return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
  4295. #else
  4296. pfn = pfn - zone->zone_start_pfn;
  4297. return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
  4298. #endif /* CONFIG_SPARSEMEM */
  4299. }
  4300. /**
  4301. * get_pageblock_flags_group - Return the requested group of flags for the pageblock_nr_pages block of pages
  4302. * @page: The page within the block of interest
  4303. * @start_bitidx: The first bit of interest to retrieve
  4304. * @end_bitidx: The last bit of interest
  4305. * returns pageblock_bits flags
  4306. */
  4307. unsigned long get_pageblock_flags_group(struct page *page,
  4308. int start_bitidx, int end_bitidx)
  4309. {
  4310. struct zone *zone;
  4311. unsigned long *bitmap;
  4312. unsigned long pfn, bitidx;
  4313. unsigned long flags = 0;
  4314. unsigned long value = 1;
  4315. zone = page_zone(page);
  4316. pfn = page_to_pfn(page);
  4317. bitmap = get_pageblock_bitmap(zone, pfn);
  4318. bitidx = pfn_to_bitidx(zone, pfn);
  4319. for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1)
  4320. if (test_bit(bitidx + start_bitidx, bitmap))
  4321. flags |= value;
  4322. return flags;
  4323. }
  4324. /**
  4325. * set_pageblock_flags_group - Set the requested group of flags for a pageblock_nr_pages block of pages
  4326. * @page: The page within the block of interest
  4327. * @start_bitidx: The first bit of interest
  4328. * @end_bitidx: The last bit of interest
  4329. * @flags: The flags to set
  4330. */
  4331. void set_pageblock_flags_group(struct page *page, unsigned long flags,
  4332. int start_bitidx, int end_bitidx)
  4333. {
  4334. struct zone *zone;
  4335. unsigned long *bitmap;
  4336. unsigned long pfn, bitidx;
  4337. unsigned long value = 1;
  4338. zone = page_zone(page);
  4339. pfn = page_to_pfn(page);
  4340. bitmap = get_pageblock_bitmap(zone, pfn);
  4341. bitidx = pfn_to_bitidx(zone, pfn);
  4342. VM_BUG_ON(pfn < zone->zone_start_pfn);
  4343. VM_BUG_ON(pfn >= zone->zone_start_pfn + zone->spanned_pages);
  4344. for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1)
  4345. if (flags & value)
  4346. __set_bit(bitidx + start_bitidx, bitmap);
  4347. else
  4348. __clear_bit(bitidx + start_bitidx, bitmap);
  4349. }
  4350. /*
  4351. * This is designed as sub function...plz see page_isolation.c also.
  4352. * set/clear page block's type to be ISOLATE.
  4353. * page allocater never alloc memory from ISOLATE block.
  4354. */
  4355. int set_migratetype_isolate(struct page *page)
  4356. {
  4357. struct zone *zone;
  4358. struct page *curr_page;
  4359. unsigned long flags, pfn, iter;
  4360. unsigned long immobile = 0;
  4361. struct memory_isolate_notify arg;
  4362. int notifier_ret;
  4363. int ret = -EBUSY;
  4364. int zone_idx;
  4365. zone = page_zone(page);
  4366. zone_idx = zone_idx(zone);
  4367. spin_lock_irqsave(&zone->lock, flags);
  4368. if (get_pageblock_migratetype(page) == MIGRATE_MOVABLE ||
  4369. zone_idx == ZONE_MOVABLE) {
  4370. ret = 0;
  4371. goto out;
  4372. }
  4373. pfn = page_to_pfn(page);
  4374. arg.start_pfn = pfn;
  4375. arg.nr_pages = pageblock_nr_pages;
  4376. arg.pages_found = 0;
  4377. /*
  4378. * It may be possible to isolate a pageblock even if the
  4379. * migratetype is not MIGRATE_MOVABLE. The memory isolation
  4380. * notifier chain is used by balloon drivers to return the
  4381. * number of pages in a range that are held by the balloon
  4382. * driver to shrink memory. If all the pages are accounted for
  4383. * by balloons, are free, or on the LRU, isolation can continue.
  4384. * Later, for example, when memory hotplug notifier runs, these
  4385. * pages reported as "can be isolated" should be isolated(freed)
  4386. * by the balloon driver through the memory notifier chain.
  4387. */
  4388. notifier_ret = memory_isolate_notify(MEM_ISOLATE_COUNT, &arg);
  4389. notifier_ret = notifier_to_errno(notifier_ret);
  4390. if (notifier_ret || !arg.pages_found)
  4391. goto out;
  4392. for (iter = pfn; iter < (pfn + pageblock_nr_pages); iter++) {
  4393. if (!pfn_valid_within(pfn))
  4394. continue;
  4395. curr_page = pfn_to_page(iter);
  4396. if (!page_count(curr_page) || PageLRU(curr_page))
  4397. continue;
  4398. immobile++;
  4399. }
  4400. if (arg.pages_found == immobile)
  4401. ret = 0;
  4402. out:
  4403. if (!ret) {
  4404. set_pageblock_migratetype(page, MIGRATE_ISOLATE);
  4405. move_freepages_block(zone, page, MIGRATE_ISOLATE);
  4406. }
  4407. spin_unlock_irqrestore(&zone->lock, flags);
  4408. if (!ret)
  4409. drain_all_pages();
  4410. return ret;
  4411. }
  4412. void unset_migratetype_isolate(struct page *page)
  4413. {
  4414. struct zone *zone;
  4415. unsigned long flags;
  4416. zone = page_zone(page);
  4417. spin_lock_irqsave(&zone->lock, flags);
  4418. if (get_pageblock_migratetype(page) != MIGRATE_ISOLATE)
  4419. goto out;
  4420. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  4421. move_freepages_block(zone, page, MIGRATE_MOVABLE);
  4422. out:
  4423. spin_unlock_irqrestore(&zone->lock, flags);
  4424. }
  4425. #ifdef CONFIG_MEMORY_HOTREMOVE
  4426. /*
  4427. * All pages in the range must be isolated before calling this.
  4428. */
  4429. void
  4430. __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
  4431. {
  4432. struct page *page;
  4433. struct zone *zone;
  4434. int order, i;
  4435. unsigned long pfn;
  4436. unsigned long flags;
  4437. /* find the first valid pfn */
  4438. for (pfn = start_pfn; pfn < end_pfn; pfn++)
  4439. if (pfn_valid(pfn))
  4440. break;
  4441. if (pfn == end_pfn)
  4442. return;
  4443. zone = page_zone(pfn_to_page(pfn));
  4444. spin_lock_irqsave(&zone->lock, flags);
  4445. pfn = start_pfn;
  4446. while (pfn < end_pfn) {
  4447. if (!pfn_valid(pfn)) {
  4448. pfn++;
  4449. continue;
  4450. }
  4451. page = pfn_to_page(pfn);
  4452. BUG_ON(page_count(page));
  4453. BUG_ON(!PageBuddy(page));
  4454. order = page_order(page);
  4455. #ifdef CONFIG_DEBUG_VM
  4456. printk(KERN_INFO "remove from free list %lx %d %lx\n",
  4457. pfn, 1 << order, end_pfn);
  4458. #endif
  4459. list_del(&page->lru);
  4460. rmv_page_order(page);
  4461. zone->free_area[order].nr_free--;
  4462. __mod_zone_page_state(zone, NR_FREE_PAGES,
  4463. - (1UL << order));
  4464. for (i = 0; i < (1 << order); i++)
  4465. SetPageReserved((page+i));
  4466. pfn += (1 << order);
  4467. }
  4468. spin_unlock_irqrestore(&zone->lock, flags);
  4469. }
  4470. #endif
  4471. #ifdef CONFIG_MEMORY_FAILURE
  4472. bool is_free_buddy_page(struct page *page)
  4473. {
  4474. struct zone *zone = page_zone(page);
  4475. unsigned long pfn = page_to_pfn(page);
  4476. unsigned long flags;
  4477. int order;
  4478. spin_lock_irqsave(&zone->lock, flags);
  4479. for (order = 0; order < MAX_ORDER; order++) {
  4480. struct page *page_head = page - (pfn & ((1 << order) - 1));
  4481. if (PageBuddy(page_head) && page_order(page_head) >= order)
  4482. break;
  4483. }
  4484. spin_unlock_irqrestore(&zone->lock, flags);
  4485. return order < MAX_ORDER;
  4486. }
  4487. #endif