page_alloc.c 149 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431
  1. /*
  2. * linux/mm/page_alloc.c
  3. *
  4. * Manages the free list, the system allocates free pages here.
  5. * Note that kmalloc() lives in slab.c
  6. *
  7. * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
  8. * Swap reorganised 29.12.95, Stephen Tweedie
  9. * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
  10. * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
  11. * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
  12. * Zone balancing, Kanoj Sarcar, SGI, Jan 2000
  13. * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
  14. * (lots of bits borrowed from Ingo Molnar & Andrew Morton)
  15. */
  16. #include <linux/stddef.h>
  17. #include <linux/mm.h>
  18. #include <linux/swap.h>
  19. #include <linux/interrupt.h>
  20. #include <linux/pagemap.h>
  21. #include <linux/jiffies.h>
  22. #include <linux/bootmem.h>
  23. #include <linux/compiler.h>
  24. #include <linux/kernel.h>
  25. #include <linux/kmemcheck.h>
  26. #include <linux/module.h>
  27. #include <linux/suspend.h>
  28. #include <linux/pagevec.h>
  29. #include <linux/blkdev.h>
  30. #include <linux/slab.h>
  31. #include <linux/oom.h>
  32. #include <linux/notifier.h>
  33. #include <linux/topology.h>
  34. #include <linux/sysctl.h>
  35. #include <linux/cpu.h>
  36. #include <linux/cpuset.h>
  37. #include <linux/memory_hotplug.h>
  38. #include <linux/nodemask.h>
  39. #include <linux/vmalloc.h>
  40. #include <linux/mempolicy.h>
  41. #include <linux/stop_machine.h>
  42. #include <linux/sort.h>
  43. #include <linux/pfn.h>
  44. #include <linux/backing-dev.h>
  45. #include <linux/fault-inject.h>
  46. #include <linux/page-isolation.h>
  47. #include <linux/page_cgroup.h>
  48. #include <linux/debugobjects.h>
  49. #include <linux/kmemleak.h>
  50. #include <linux/memory.h>
  51. #include <linux/compaction.h>
  52. #include <trace/events/kmem.h>
  53. #include <linux/ftrace_event.h>
  54. #include <asm/tlbflush.h>
  55. #include <asm/div64.h>
  56. #include "internal.h"
  57. /*
  58. * Array of node states.
  59. */
  60. nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
  61. [N_POSSIBLE] = NODE_MASK_ALL,
  62. [N_ONLINE] = { { [0] = 1UL } },
  63. #ifndef CONFIG_NUMA
  64. [N_NORMAL_MEMORY] = { { [0] = 1UL } },
  65. #ifdef CONFIG_HIGHMEM
  66. [N_HIGH_MEMORY] = { { [0] = 1UL } },
  67. #endif
  68. [N_CPU] = { { [0] = 1UL } },
  69. #endif /* NUMA */
  70. };
  71. EXPORT_SYMBOL(node_states);
  72. unsigned long totalram_pages __read_mostly;
  73. unsigned long totalreserve_pages __read_mostly;
  74. int percpu_pagelist_fraction;
  75. gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
  76. #ifdef CONFIG_PM_SLEEP
  77. /*
  78. * The following functions are used by the suspend/hibernate code to temporarily
  79. * change gfp_allowed_mask in order to avoid using I/O during memory allocations
  80. * while devices are suspended. To avoid races with the suspend/hibernate code,
  81. * they should always be called with pm_mutex held (gfp_allowed_mask also should
  82. * only be modified with pm_mutex held, unless the suspend/hibernate code is
  83. * guaranteed not to run in parallel with that modification).
  84. */
  85. void set_gfp_allowed_mask(gfp_t mask)
  86. {
  87. WARN_ON(!mutex_is_locked(&pm_mutex));
  88. gfp_allowed_mask = mask;
  89. }
  90. gfp_t clear_gfp_allowed_mask(gfp_t mask)
  91. {
  92. gfp_t ret = gfp_allowed_mask;
  93. WARN_ON(!mutex_is_locked(&pm_mutex));
  94. gfp_allowed_mask &= ~mask;
  95. return ret;
  96. }
  97. #endif /* CONFIG_PM_SLEEP */
  98. #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
  99. int pageblock_order __read_mostly;
  100. #endif
  101. static void __free_pages_ok(struct page *page, unsigned int order);
  102. /*
  103. * results with 256, 32 in the lowmem_reserve sysctl:
  104. * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
  105. * 1G machine -> (16M dma, 784M normal, 224M high)
  106. * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
  107. * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
  108. * HIGHMEM allocation will (224M+784M)/256 of ram reserved in ZONE_DMA
  109. *
  110. * TBD: should special case ZONE_DMA32 machines here - in those we normally
  111. * don't need any ZONE_NORMAL reservation
  112. */
  113. int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = {
  114. #ifdef CONFIG_ZONE_DMA
  115. 256,
  116. #endif
  117. #ifdef CONFIG_ZONE_DMA32
  118. 256,
  119. #endif
  120. #ifdef CONFIG_HIGHMEM
  121. 32,
  122. #endif
  123. 32,
  124. };
  125. EXPORT_SYMBOL(totalram_pages);
  126. static char * const zone_names[MAX_NR_ZONES] = {
  127. #ifdef CONFIG_ZONE_DMA
  128. "DMA",
  129. #endif
  130. #ifdef CONFIG_ZONE_DMA32
  131. "DMA32",
  132. #endif
  133. "Normal",
  134. #ifdef CONFIG_HIGHMEM
  135. "HighMem",
  136. #endif
  137. "Movable",
  138. };
  139. int min_free_kbytes = 1024;
  140. static unsigned long __meminitdata nr_kernel_pages;
  141. static unsigned long __meminitdata nr_all_pages;
  142. static unsigned long __meminitdata dma_reserve;
  143. #ifdef CONFIG_ARCH_POPULATES_NODE_MAP
  144. /*
  145. * MAX_ACTIVE_REGIONS determines the maximum number of distinct
  146. * ranges of memory (RAM) that may be registered with add_active_range().
  147. * Ranges passed to add_active_range() will be merged if possible
  148. * so the number of times add_active_range() can be called is
  149. * related to the number of nodes and the number of holes
  150. */
  151. #ifdef CONFIG_MAX_ACTIVE_REGIONS
  152. /* Allow an architecture to set MAX_ACTIVE_REGIONS to save memory */
  153. #define MAX_ACTIVE_REGIONS CONFIG_MAX_ACTIVE_REGIONS
  154. #else
  155. #if MAX_NUMNODES >= 32
  156. /* If there can be many nodes, allow up to 50 holes per node */
  157. #define MAX_ACTIVE_REGIONS (MAX_NUMNODES*50)
  158. #else
  159. /* By default, allow up to 256 distinct regions */
  160. #define MAX_ACTIVE_REGIONS 256
  161. #endif
  162. #endif
  163. static struct node_active_region __meminitdata early_node_map[MAX_ACTIVE_REGIONS];
  164. static int __meminitdata nr_nodemap_entries;
  165. static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES];
  166. static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES];
  167. static unsigned long __initdata required_kernelcore;
  168. static unsigned long __initdata required_movablecore;
  169. static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES];
  170. /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
  171. int movable_zone;
  172. EXPORT_SYMBOL(movable_zone);
  173. #endif /* CONFIG_ARCH_POPULATES_NODE_MAP */
  174. #if MAX_NUMNODES > 1
  175. int nr_node_ids __read_mostly = MAX_NUMNODES;
  176. int nr_online_nodes __read_mostly = 1;
  177. EXPORT_SYMBOL(nr_node_ids);
  178. EXPORT_SYMBOL(nr_online_nodes);
  179. #endif
  180. int page_group_by_mobility_disabled __read_mostly;
  181. static void set_pageblock_migratetype(struct page *page, int migratetype)
  182. {
  183. if (unlikely(page_group_by_mobility_disabled))
  184. migratetype = MIGRATE_UNMOVABLE;
  185. set_pageblock_flags_group(page, (unsigned long)migratetype,
  186. PB_migrate, PB_migrate_end);
  187. }
  188. bool oom_killer_disabled __read_mostly;
  189. #ifdef CONFIG_DEBUG_VM
  190. static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
  191. {
  192. int ret = 0;
  193. unsigned seq;
  194. unsigned long pfn = page_to_pfn(page);
  195. do {
  196. seq = zone_span_seqbegin(zone);
  197. if (pfn >= zone->zone_start_pfn + zone->spanned_pages)
  198. ret = 1;
  199. else if (pfn < zone->zone_start_pfn)
  200. ret = 1;
  201. } while (zone_span_seqretry(zone, seq));
  202. return ret;
  203. }
  204. static int page_is_consistent(struct zone *zone, struct page *page)
  205. {
  206. if (!pfn_valid_within(page_to_pfn(page)))
  207. return 0;
  208. if (zone != page_zone(page))
  209. return 0;
  210. return 1;
  211. }
  212. /*
  213. * Temporary debugging check for pages not lying within a given zone.
  214. */
  215. static int bad_range(struct zone *zone, struct page *page)
  216. {
  217. if (page_outside_zone_boundaries(zone, page))
  218. return 1;
  219. if (!page_is_consistent(zone, page))
  220. return 1;
  221. return 0;
  222. }
  223. #else
  224. static inline int bad_range(struct zone *zone, struct page *page)
  225. {
  226. return 0;
  227. }
  228. #endif
  229. static void bad_page(struct page *page)
  230. {
  231. static unsigned long resume;
  232. static unsigned long nr_shown;
  233. static unsigned long nr_unshown;
  234. /* Don't complain about poisoned pages */
  235. if (PageHWPoison(page)) {
  236. __ClearPageBuddy(page);
  237. return;
  238. }
  239. /*
  240. * Allow a burst of 60 reports, then keep quiet for that minute;
  241. * or allow a steady drip of one report per second.
  242. */
  243. if (nr_shown == 60) {
  244. if (time_before(jiffies, resume)) {
  245. nr_unshown++;
  246. goto out;
  247. }
  248. if (nr_unshown) {
  249. printk(KERN_ALERT
  250. "BUG: Bad page state: %lu messages suppressed\n",
  251. nr_unshown);
  252. nr_unshown = 0;
  253. }
  254. nr_shown = 0;
  255. }
  256. if (nr_shown++ == 0)
  257. resume = jiffies + 60 * HZ;
  258. printk(KERN_ALERT "BUG: Bad page state in process %s pfn:%05lx\n",
  259. current->comm, page_to_pfn(page));
  260. dump_page(page);
  261. dump_stack();
  262. out:
  263. /* Leave bad fields for debug, except PageBuddy could make trouble */
  264. __ClearPageBuddy(page);
  265. add_taint(TAINT_BAD_PAGE);
  266. }
  267. /*
  268. * Higher-order pages are called "compound pages". They are structured thusly:
  269. *
  270. * The first PAGE_SIZE page is called the "head page".
  271. *
  272. * The remaining PAGE_SIZE pages are called "tail pages".
  273. *
  274. * All pages have PG_compound set. All pages have their ->private pointing at
  275. * the head page (even the head page has this).
  276. *
  277. * The first tail page's ->lru.next holds the address of the compound page's
  278. * put_page() function. Its ->lru.prev holds the order of allocation.
  279. * This usage means that zero-order pages may not be compound.
  280. */
  281. static void free_compound_page(struct page *page)
  282. {
  283. __free_pages_ok(page, compound_order(page));
  284. }
  285. void prep_compound_page(struct page *page, unsigned long order)
  286. {
  287. int i;
  288. int nr_pages = 1 << order;
  289. set_compound_page_dtor(page, free_compound_page);
  290. set_compound_order(page, order);
  291. __SetPageHead(page);
  292. for (i = 1; i < nr_pages; i++) {
  293. struct page *p = page + i;
  294. __SetPageTail(p);
  295. p->first_page = page;
  296. }
  297. }
  298. static int destroy_compound_page(struct page *page, unsigned long order)
  299. {
  300. int i;
  301. int nr_pages = 1 << order;
  302. int bad = 0;
  303. if (unlikely(compound_order(page) != order) ||
  304. unlikely(!PageHead(page))) {
  305. bad_page(page);
  306. bad++;
  307. }
  308. __ClearPageHead(page);
  309. for (i = 1; i < nr_pages; i++) {
  310. struct page *p = page + i;
  311. if (unlikely(!PageTail(p) || (p->first_page != page))) {
  312. bad_page(page);
  313. bad++;
  314. }
  315. __ClearPageTail(p);
  316. }
  317. return bad;
  318. }
  319. static inline void prep_zero_page(struct page *page, int order, gfp_t gfp_flags)
  320. {
  321. int i;
  322. /*
  323. * clear_highpage() will use KM_USER0, so it's a bug to use __GFP_ZERO
  324. * and __GFP_HIGHMEM from hard or soft interrupt context.
  325. */
  326. VM_BUG_ON((gfp_flags & __GFP_HIGHMEM) && in_interrupt());
  327. for (i = 0; i < (1 << order); i++)
  328. clear_highpage(page + i);
  329. }
  330. static inline void set_page_order(struct page *page, int order)
  331. {
  332. set_page_private(page, order);
  333. __SetPageBuddy(page);
  334. }
  335. static inline void rmv_page_order(struct page *page)
  336. {
  337. __ClearPageBuddy(page);
  338. set_page_private(page, 0);
  339. }
  340. /*
  341. * Locate the struct page for both the matching buddy in our
  342. * pair (buddy1) and the combined O(n+1) page they form (page).
  343. *
  344. * 1) Any buddy B1 will have an order O twin B2 which satisfies
  345. * the following equation:
  346. * B2 = B1 ^ (1 << O)
  347. * For example, if the starting buddy (buddy2) is #8 its order
  348. * 1 buddy is #10:
  349. * B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10
  350. *
  351. * 2) Any buddy B will have an order O+1 parent P which
  352. * satisfies the following equation:
  353. * P = B & ~(1 << O)
  354. *
  355. * Assumption: *_mem_map is contiguous at least up to MAX_ORDER
  356. */
  357. static inline struct page *
  358. __page_find_buddy(struct page *page, unsigned long page_idx, unsigned int order)
  359. {
  360. unsigned long buddy_idx = page_idx ^ (1 << order);
  361. return page + (buddy_idx - page_idx);
  362. }
  363. static inline unsigned long
  364. __find_combined_index(unsigned long page_idx, unsigned int order)
  365. {
  366. return (page_idx & ~(1 << order));
  367. }
  368. /*
  369. * This function checks whether a page is free && is the buddy
  370. * we can do coalesce a page and its buddy if
  371. * (a) the buddy is not in a hole &&
  372. * (b) the buddy is in the buddy system &&
  373. * (c) a page and its buddy have the same order &&
  374. * (d) a page and its buddy are in the same zone.
  375. *
  376. * For recording whether a page is in the buddy system, we use PG_buddy.
  377. * Setting, clearing, and testing PG_buddy is serialized by zone->lock.
  378. *
  379. * For recording page's order, we use page_private(page).
  380. */
  381. static inline int page_is_buddy(struct page *page, struct page *buddy,
  382. int order)
  383. {
  384. if (!pfn_valid_within(page_to_pfn(buddy)))
  385. return 0;
  386. if (page_zone_id(page) != page_zone_id(buddy))
  387. return 0;
  388. if (PageBuddy(buddy) && page_order(buddy) == order) {
  389. VM_BUG_ON(page_count(buddy) != 0);
  390. return 1;
  391. }
  392. return 0;
  393. }
  394. /*
  395. * Freeing function for a buddy system allocator.
  396. *
  397. * The concept of a buddy system is to maintain direct-mapped table
  398. * (containing bit values) for memory blocks of various "orders".
  399. * The bottom level table contains the map for the smallest allocatable
  400. * units of memory (here, pages), and each level above it describes
  401. * pairs of units from the levels below, hence, "buddies".
  402. * At a high level, all that happens here is marking the table entry
  403. * at the bottom level available, and propagating the changes upward
  404. * as necessary, plus some accounting needed to play nicely with other
  405. * parts of the VM system.
  406. * At each level, we keep a list of pages, which are heads of continuous
  407. * free pages of length of (1 << order) and marked with PG_buddy. Page's
  408. * order is recorded in page_private(page) field.
  409. * So when we are allocating or freeing one, we can derive the state of the
  410. * other. That is, if we allocate a small block, and both were
  411. * free, the remainder of the region must be split into blocks.
  412. * If a block is freed, and its buddy is also free, then this
  413. * triggers coalescing into a block of larger size.
  414. *
  415. * -- wli
  416. */
  417. static inline void __free_one_page(struct page *page,
  418. struct zone *zone, unsigned int order,
  419. int migratetype)
  420. {
  421. unsigned long page_idx;
  422. unsigned long combined_idx;
  423. struct page *buddy;
  424. if (unlikely(PageCompound(page)))
  425. if (unlikely(destroy_compound_page(page, order)))
  426. return;
  427. VM_BUG_ON(migratetype == -1);
  428. page_idx = page_to_pfn(page) & ((1 << MAX_ORDER) - 1);
  429. VM_BUG_ON(page_idx & ((1 << order) - 1));
  430. VM_BUG_ON(bad_range(zone, page));
  431. while (order < MAX_ORDER-1) {
  432. buddy = __page_find_buddy(page, page_idx, order);
  433. if (!page_is_buddy(page, buddy, order))
  434. break;
  435. /* Our buddy is free, merge with it and move up one order. */
  436. list_del(&buddy->lru);
  437. zone->free_area[order].nr_free--;
  438. rmv_page_order(buddy);
  439. combined_idx = __find_combined_index(page_idx, order);
  440. page = page + (combined_idx - page_idx);
  441. page_idx = combined_idx;
  442. order++;
  443. }
  444. set_page_order(page, order);
  445. /*
  446. * If this is not the largest possible page, check if the buddy
  447. * of the next-highest order is free. If it is, it's possible
  448. * that pages are being freed that will coalesce soon. In case,
  449. * that is happening, add the free page to the tail of the list
  450. * so it's less likely to be used soon and more likely to be merged
  451. * as a higher order page
  452. */
  453. if ((order < MAX_ORDER-1) && pfn_valid_within(page_to_pfn(buddy))) {
  454. struct page *higher_page, *higher_buddy;
  455. combined_idx = __find_combined_index(page_idx, order);
  456. higher_page = page + combined_idx - page_idx;
  457. higher_buddy = __page_find_buddy(higher_page, combined_idx, order + 1);
  458. if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
  459. list_add_tail(&page->lru,
  460. &zone->free_area[order].free_list[migratetype]);
  461. goto out;
  462. }
  463. }
  464. list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
  465. out:
  466. zone->free_area[order].nr_free++;
  467. }
  468. /*
  469. * free_page_mlock() -- clean up attempts to free and mlocked() page.
  470. * Page should not be on lru, so no need to fix that up.
  471. * free_pages_check() will verify...
  472. */
  473. static inline void free_page_mlock(struct page *page)
  474. {
  475. __dec_zone_page_state(page, NR_MLOCK);
  476. __count_vm_event(UNEVICTABLE_MLOCKFREED);
  477. }
  478. static inline int free_pages_check(struct page *page)
  479. {
  480. if (unlikely(page_mapcount(page) |
  481. (page->mapping != NULL) |
  482. (atomic_read(&page->_count) != 0) |
  483. (page->flags & PAGE_FLAGS_CHECK_AT_FREE))) {
  484. bad_page(page);
  485. return 1;
  486. }
  487. if (page->flags & PAGE_FLAGS_CHECK_AT_PREP)
  488. page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
  489. return 0;
  490. }
  491. /*
  492. * Frees a number of pages from the PCP lists
  493. * Assumes all pages on list are in same zone, and of same order.
  494. * count is the number of pages to free.
  495. *
  496. * If the zone was previously in an "all pages pinned" state then look to
  497. * see if this freeing clears that state.
  498. *
  499. * And clear the zone's pages_scanned counter, to hold off the "all pages are
  500. * pinned" detection logic.
  501. */
  502. static void free_pcppages_bulk(struct zone *zone, int count,
  503. struct per_cpu_pages *pcp)
  504. {
  505. int migratetype = 0;
  506. int batch_free = 0;
  507. spin_lock(&zone->lock);
  508. zone->all_unreclaimable = 0;
  509. zone->pages_scanned = 0;
  510. __mod_zone_page_state(zone, NR_FREE_PAGES, count);
  511. while (count) {
  512. struct page *page;
  513. struct list_head *list;
  514. /*
  515. * Remove pages from lists in a round-robin fashion. A
  516. * batch_free count is maintained that is incremented when an
  517. * empty list is encountered. This is so more pages are freed
  518. * off fuller lists instead of spinning excessively around empty
  519. * lists
  520. */
  521. do {
  522. batch_free++;
  523. if (++migratetype == MIGRATE_PCPTYPES)
  524. migratetype = 0;
  525. list = &pcp->lists[migratetype];
  526. } while (list_empty(list));
  527. do {
  528. page = list_entry(list->prev, struct page, lru);
  529. /* must delete as __free_one_page list manipulates */
  530. list_del(&page->lru);
  531. /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */
  532. __free_one_page(page, zone, 0, page_private(page));
  533. trace_mm_page_pcpu_drain(page, 0, page_private(page));
  534. } while (--count && --batch_free && !list_empty(list));
  535. }
  536. spin_unlock(&zone->lock);
  537. }
  538. static void free_one_page(struct zone *zone, struct page *page, int order,
  539. int migratetype)
  540. {
  541. spin_lock(&zone->lock);
  542. zone->all_unreclaimable = 0;
  543. zone->pages_scanned = 0;
  544. __mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order);
  545. __free_one_page(page, zone, order, migratetype);
  546. spin_unlock(&zone->lock);
  547. }
  548. static bool free_pages_prepare(struct page *page, unsigned int order)
  549. {
  550. int i;
  551. int bad = 0;
  552. trace_mm_page_free_direct(page, order);
  553. kmemcheck_free_shadow(page, order);
  554. for (i = 0; i < (1 << order); i++) {
  555. struct page *pg = page + i;
  556. if (PageAnon(pg))
  557. pg->mapping = NULL;
  558. bad += free_pages_check(pg);
  559. }
  560. if (bad)
  561. return false;
  562. if (!PageHighMem(page)) {
  563. debug_check_no_locks_freed(page_address(page),PAGE_SIZE<<order);
  564. debug_check_no_obj_freed(page_address(page),
  565. PAGE_SIZE << order);
  566. }
  567. arch_free_page(page, order);
  568. kernel_map_pages(page, 1 << order, 0);
  569. return true;
  570. }
  571. static void __free_pages_ok(struct page *page, unsigned int order)
  572. {
  573. unsigned long flags;
  574. int wasMlocked = __TestClearPageMlocked(page);
  575. if (!free_pages_prepare(page, order))
  576. return;
  577. local_irq_save(flags);
  578. if (unlikely(wasMlocked))
  579. free_page_mlock(page);
  580. __count_vm_events(PGFREE, 1 << order);
  581. free_one_page(page_zone(page), page, order,
  582. get_pageblock_migratetype(page));
  583. local_irq_restore(flags);
  584. }
  585. /*
  586. * permit the bootmem allocator to evade page validation on high-order frees
  587. */
  588. void __meminit __free_pages_bootmem(struct page *page, unsigned int order)
  589. {
  590. if (order == 0) {
  591. __ClearPageReserved(page);
  592. set_page_count(page, 0);
  593. set_page_refcounted(page);
  594. __free_page(page);
  595. } else {
  596. int loop;
  597. prefetchw(page);
  598. for (loop = 0; loop < BITS_PER_LONG; loop++) {
  599. struct page *p = &page[loop];
  600. if (loop + 1 < BITS_PER_LONG)
  601. prefetchw(p + 1);
  602. __ClearPageReserved(p);
  603. set_page_count(p, 0);
  604. }
  605. set_page_refcounted(page);
  606. __free_pages(page, order);
  607. }
  608. }
  609. /*
  610. * The order of subdivision here is critical for the IO subsystem.
  611. * Please do not alter this order without good reasons and regression
  612. * testing. Specifically, as large blocks of memory are subdivided,
  613. * the order in which smaller blocks are delivered depends on the order
  614. * they're subdivided in this function. This is the primary factor
  615. * influencing the order in which pages are delivered to the IO
  616. * subsystem according to empirical testing, and this is also justified
  617. * by considering the behavior of a buddy system containing a single
  618. * large block of memory acted on by a series of small allocations.
  619. * This behavior is a critical factor in sglist merging's success.
  620. *
  621. * -- wli
  622. */
  623. static inline void expand(struct zone *zone, struct page *page,
  624. int low, int high, struct free_area *area,
  625. int migratetype)
  626. {
  627. unsigned long size = 1 << high;
  628. while (high > low) {
  629. area--;
  630. high--;
  631. size >>= 1;
  632. VM_BUG_ON(bad_range(zone, &page[size]));
  633. list_add(&page[size].lru, &area->free_list[migratetype]);
  634. area->nr_free++;
  635. set_page_order(&page[size], high);
  636. }
  637. }
  638. /*
  639. * This page is about to be returned from the page allocator
  640. */
  641. static inline int check_new_page(struct page *page)
  642. {
  643. if (unlikely(page_mapcount(page) |
  644. (page->mapping != NULL) |
  645. (atomic_read(&page->_count) != 0) |
  646. (page->flags & PAGE_FLAGS_CHECK_AT_PREP))) {
  647. bad_page(page);
  648. return 1;
  649. }
  650. return 0;
  651. }
  652. static int prep_new_page(struct page *page, int order, gfp_t gfp_flags)
  653. {
  654. int i;
  655. for (i = 0; i < (1 << order); i++) {
  656. struct page *p = page + i;
  657. if (unlikely(check_new_page(p)))
  658. return 1;
  659. }
  660. set_page_private(page, 0);
  661. set_page_refcounted(page);
  662. arch_alloc_page(page, order);
  663. kernel_map_pages(page, 1 << order, 1);
  664. if (gfp_flags & __GFP_ZERO)
  665. prep_zero_page(page, order, gfp_flags);
  666. if (order && (gfp_flags & __GFP_COMP))
  667. prep_compound_page(page, order);
  668. return 0;
  669. }
  670. /*
  671. * Go through the free lists for the given migratetype and remove
  672. * the smallest available page from the freelists
  673. */
  674. static inline
  675. struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
  676. int migratetype)
  677. {
  678. unsigned int current_order;
  679. struct free_area * area;
  680. struct page *page;
  681. /* Find a page of the appropriate size in the preferred list */
  682. for (current_order = order; current_order < MAX_ORDER; ++current_order) {
  683. area = &(zone->free_area[current_order]);
  684. if (list_empty(&area->free_list[migratetype]))
  685. continue;
  686. page = list_entry(area->free_list[migratetype].next,
  687. struct page, lru);
  688. list_del(&page->lru);
  689. rmv_page_order(page);
  690. area->nr_free--;
  691. expand(zone, page, order, current_order, area, migratetype);
  692. return page;
  693. }
  694. return NULL;
  695. }
  696. /*
  697. * This array describes the order lists are fallen back to when
  698. * the free lists for the desirable migrate type are depleted
  699. */
  700. static int fallbacks[MIGRATE_TYPES][MIGRATE_TYPES-1] = {
  701. [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE },
  702. [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE },
  703. [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },
  704. [MIGRATE_RESERVE] = { MIGRATE_RESERVE, MIGRATE_RESERVE, MIGRATE_RESERVE }, /* Never used */
  705. };
  706. /*
  707. * Move the free pages in a range to the free lists of the requested type.
  708. * Note that start_page and end_pages are not aligned on a pageblock
  709. * boundary. If alignment is required, use move_freepages_block()
  710. */
  711. static int move_freepages(struct zone *zone,
  712. struct page *start_page, struct page *end_page,
  713. int migratetype)
  714. {
  715. struct page *page;
  716. unsigned long order;
  717. int pages_moved = 0;
  718. #ifndef CONFIG_HOLES_IN_ZONE
  719. /*
  720. * page_zone is not safe to call in this context when
  721. * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant
  722. * anyway as we check zone boundaries in move_freepages_block().
  723. * Remove at a later date when no bug reports exist related to
  724. * grouping pages by mobility
  725. */
  726. BUG_ON(page_zone(start_page) != page_zone(end_page));
  727. #endif
  728. for (page = start_page; page <= end_page;) {
  729. /* Make sure we are not inadvertently changing nodes */
  730. VM_BUG_ON(page_to_nid(page) != zone_to_nid(zone));
  731. if (!pfn_valid_within(page_to_pfn(page))) {
  732. page++;
  733. continue;
  734. }
  735. if (!PageBuddy(page)) {
  736. page++;
  737. continue;
  738. }
  739. order = page_order(page);
  740. list_del(&page->lru);
  741. list_add(&page->lru,
  742. &zone->free_area[order].free_list[migratetype]);
  743. page += 1 << order;
  744. pages_moved += 1 << order;
  745. }
  746. return pages_moved;
  747. }
  748. static int move_freepages_block(struct zone *zone, struct page *page,
  749. int migratetype)
  750. {
  751. unsigned long start_pfn, end_pfn;
  752. struct page *start_page, *end_page;
  753. start_pfn = page_to_pfn(page);
  754. start_pfn = start_pfn & ~(pageblock_nr_pages-1);
  755. start_page = pfn_to_page(start_pfn);
  756. end_page = start_page + pageblock_nr_pages - 1;
  757. end_pfn = start_pfn + pageblock_nr_pages - 1;
  758. /* Do not cross zone boundaries */
  759. if (start_pfn < zone->zone_start_pfn)
  760. start_page = page;
  761. if (end_pfn >= zone->zone_start_pfn + zone->spanned_pages)
  762. return 0;
  763. return move_freepages(zone, start_page, end_page, migratetype);
  764. }
  765. static void change_pageblock_range(struct page *pageblock_page,
  766. int start_order, int migratetype)
  767. {
  768. int nr_pageblocks = 1 << (start_order - pageblock_order);
  769. while (nr_pageblocks--) {
  770. set_pageblock_migratetype(pageblock_page, migratetype);
  771. pageblock_page += pageblock_nr_pages;
  772. }
  773. }
  774. /* Remove an element from the buddy allocator from the fallback list */
  775. static inline struct page *
  776. __rmqueue_fallback(struct zone *zone, int order, int start_migratetype)
  777. {
  778. struct free_area * area;
  779. int current_order;
  780. struct page *page;
  781. int migratetype, i;
  782. /* Find the largest possible block of pages in the other list */
  783. for (current_order = MAX_ORDER-1; current_order >= order;
  784. --current_order) {
  785. for (i = 0; i < MIGRATE_TYPES - 1; i++) {
  786. migratetype = fallbacks[start_migratetype][i];
  787. /* MIGRATE_RESERVE handled later if necessary */
  788. if (migratetype == MIGRATE_RESERVE)
  789. continue;
  790. area = &(zone->free_area[current_order]);
  791. if (list_empty(&area->free_list[migratetype]))
  792. continue;
  793. page = list_entry(area->free_list[migratetype].next,
  794. struct page, lru);
  795. area->nr_free--;
  796. /*
  797. * If breaking a large block of pages, move all free
  798. * pages to the preferred allocation list. If falling
  799. * back for a reclaimable kernel allocation, be more
  800. * agressive about taking ownership of free pages
  801. */
  802. if (unlikely(current_order >= (pageblock_order >> 1)) ||
  803. start_migratetype == MIGRATE_RECLAIMABLE ||
  804. page_group_by_mobility_disabled) {
  805. unsigned long pages;
  806. pages = move_freepages_block(zone, page,
  807. start_migratetype);
  808. /* Claim the whole block if over half of it is free */
  809. if (pages >= (1 << (pageblock_order-1)) ||
  810. page_group_by_mobility_disabled)
  811. set_pageblock_migratetype(page,
  812. start_migratetype);
  813. migratetype = start_migratetype;
  814. }
  815. /* Remove the page from the freelists */
  816. list_del(&page->lru);
  817. rmv_page_order(page);
  818. /* Take ownership for orders >= pageblock_order */
  819. if (current_order >= pageblock_order)
  820. change_pageblock_range(page, current_order,
  821. start_migratetype);
  822. expand(zone, page, order, current_order, area, migratetype);
  823. trace_mm_page_alloc_extfrag(page, order, current_order,
  824. start_migratetype, migratetype);
  825. return page;
  826. }
  827. }
  828. return NULL;
  829. }
  830. /*
  831. * Do the hard work of removing an element from the buddy allocator.
  832. * Call me with the zone->lock already held.
  833. */
  834. static struct page *__rmqueue(struct zone *zone, unsigned int order,
  835. int migratetype)
  836. {
  837. struct page *page;
  838. retry_reserve:
  839. page = __rmqueue_smallest(zone, order, migratetype);
  840. if (unlikely(!page) && migratetype != MIGRATE_RESERVE) {
  841. page = __rmqueue_fallback(zone, order, migratetype);
  842. /*
  843. * Use MIGRATE_RESERVE rather than fail an allocation. goto
  844. * is used because __rmqueue_smallest is an inline function
  845. * and we want just one call site
  846. */
  847. if (!page) {
  848. migratetype = MIGRATE_RESERVE;
  849. goto retry_reserve;
  850. }
  851. }
  852. trace_mm_page_alloc_zone_locked(page, order, migratetype);
  853. return page;
  854. }
  855. /*
  856. * Obtain a specified number of elements from the buddy allocator, all under
  857. * a single hold of the lock, for efficiency. Add them to the supplied list.
  858. * Returns the number of new pages which were placed at *list.
  859. */
  860. static int rmqueue_bulk(struct zone *zone, unsigned int order,
  861. unsigned long count, struct list_head *list,
  862. int migratetype, int cold)
  863. {
  864. int i;
  865. spin_lock(&zone->lock);
  866. for (i = 0; i < count; ++i) {
  867. struct page *page = __rmqueue(zone, order, migratetype);
  868. if (unlikely(page == NULL))
  869. break;
  870. /*
  871. * Split buddy pages returned by expand() are received here
  872. * in physical page order. The page is added to the callers and
  873. * list and the list head then moves forward. From the callers
  874. * perspective, the linked list is ordered by page number in
  875. * some conditions. This is useful for IO devices that can
  876. * merge IO requests if the physical pages are ordered
  877. * properly.
  878. */
  879. if (likely(cold == 0))
  880. list_add(&page->lru, list);
  881. else
  882. list_add_tail(&page->lru, list);
  883. set_page_private(page, migratetype);
  884. list = &page->lru;
  885. }
  886. __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
  887. spin_unlock(&zone->lock);
  888. return i;
  889. }
  890. #ifdef CONFIG_NUMA
  891. /*
  892. * Called from the vmstat counter updater to drain pagesets of this
  893. * currently executing processor on remote nodes after they have
  894. * expired.
  895. *
  896. * Note that this function must be called with the thread pinned to
  897. * a single processor.
  898. */
  899. void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
  900. {
  901. unsigned long flags;
  902. int to_drain;
  903. local_irq_save(flags);
  904. if (pcp->count >= pcp->batch)
  905. to_drain = pcp->batch;
  906. else
  907. to_drain = pcp->count;
  908. free_pcppages_bulk(zone, to_drain, pcp);
  909. pcp->count -= to_drain;
  910. local_irq_restore(flags);
  911. }
  912. #endif
  913. /*
  914. * Drain pages of the indicated processor.
  915. *
  916. * The processor must either be the current processor and the
  917. * thread pinned to the current processor or a processor that
  918. * is not online.
  919. */
  920. static void drain_pages(unsigned int cpu)
  921. {
  922. unsigned long flags;
  923. struct zone *zone;
  924. for_each_populated_zone(zone) {
  925. struct per_cpu_pageset *pset;
  926. struct per_cpu_pages *pcp;
  927. local_irq_save(flags);
  928. pset = per_cpu_ptr(zone->pageset, cpu);
  929. pcp = &pset->pcp;
  930. free_pcppages_bulk(zone, pcp->count, pcp);
  931. pcp->count = 0;
  932. local_irq_restore(flags);
  933. }
  934. }
  935. /*
  936. * Spill all of this CPU's per-cpu pages back into the buddy allocator.
  937. */
  938. void drain_local_pages(void *arg)
  939. {
  940. drain_pages(smp_processor_id());
  941. }
  942. /*
  943. * Spill all the per-cpu pages from all CPUs back into the buddy allocator
  944. */
  945. void drain_all_pages(void)
  946. {
  947. on_each_cpu(drain_local_pages, NULL, 1);
  948. }
  949. #ifdef CONFIG_HIBERNATION
  950. void mark_free_pages(struct zone *zone)
  951. {
  952. unsigned long pfn, max_zone_pfn;
  953. unsigned long flags;
  954. int order, t;
  955. struct list_head *curr;
  956. if (!zone->spanned_pages)
  957. return;
  958. spin_lock_irqsave(&zone->lock, flags);
  959. max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
  960. for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
  961. if (pfn_valid(pfn)) {
  962. struct page *page = pfn_to_page(pfn);
  963. if (!swsusp_page_is_forbidden(page))
  964. swsusp_unset_page_free(page);
  965. }
  966. for_each_migratetype_order(order, t) {
  967. list_for_each(curr, &zone->free_area[order].free_list[t]) {
  968. unsigned long i;
  969. pfn = page_to_pfn(list_entry(curr, struct page, lru));
  970. for (i = 0; i < (1UL << order); i++)
  971. swsusp_set_page_free(pfn_to_page(pfn + i));
  972. }
  973. }
  974. spin_unlock_irqrestore(&zone->lock, flags);
  975. }
  976. #endif /* CONFIG_PM */
  977. /*
  978. * Free a 0-order page
  979. * cold == 1 ? free a cold page : free a hot page
  980. */
  981. void free_hot_cold_page(struct page *page, int cold)
  982. {
  983. struct zone *zone = page_zone(page);
  984. struct per_cpu_pages *pcp;
  985. unsigned long flags;
  986. int migratetype;
  987. int wasMlocked = __TestClearPageMlocked(page);
  988. if (!free_pages_prepare(page, 0))
  989. return;
  990. migratetype = get_pageblock_migratetype(page);
  991. set_page_private(page, migratetype);
  992. local_irq_save(flags);
  993. if (unlikely(wasMlocked))
  994. free_page_mlock(page);
  995. __count_vm_event(PGFREE);
  996. /*
  997. * We only track unmovable, reclaimable and movable on pcp lists.
  998. * Free ISOLATE pages back to the allocator because they are being
  999. * offlined but treat RESERVE as movable pages so we can get those
  1000. * areas back if necessary. Otherwise, we may have to free
  1001. * excessively into the page allocator
  1002. */
  1003. if (migratetype >= MIGRATE_PCPTYPES) {
  1004. if (unlikely(migratetype == MIGRATE_ISOLATE)) {
  1005. free_one_page(zone, page, 0, migratetype);
  1006. goto out;
  1007. }
  1008. migratetype = MIGRATE_MOVABLE;
  1009. }
  1010. pcp = &this_cpu_ptr(zone->pageset)->pcp;
  1011. if (cold)
  1012. list_add_tail(&page->lru, &pcp->lists[migratetype]);
  1013. else
  1014. list_add(&page->lru, &pcp->lists[migratetype]);
  1015. pcp->count++;
  1016. if (pcp->count >= pcp->high) {
  1017. free_pcppages_bulk(zone, pcp->batch, pcp);
  1018. pcp->count -= pcp->batch;
  1019. }
  1020. out:
  1021. local_irq_restore(flags);
  1022. }
  1023. /*
  1024. * split_page takes a non-compound higher-order page, and splits it into
  1025. * n (1<<order) sub-pages: page[0..n]
  1026. * Each sub-page must be freed individually.
  1027. *
  1028. * Note: this is probably too low level an operation for use in drivers.
  1029. * Please consult with lkml before using this in your driver.
  1030. */
  1031. void split_page(struct page *page, unsigned int order)
  1032. {
  1033. int i;
  1034. VM_BUG_ON(PageCompound(page));
  1035. VM_BUG_ON(!page_count(page));
  1036. #ifdef CONFIG_KMEMCHECK
  1037. /*
  1038. * Split shadow pages too, because free(page[0]) would
  1039. * otherwise free the whole shadow.
  1040. */
  1041. if (kmemcheck_page_is_tracked(page))
  1042. split_page(virt_to_page(page[0].shadow), order);
  1043. #endif
  1044. for (i = 1; i < (1 << order); i++)
  1045. set_page_refcounted(page + i);
  1046. }
  1047. /*
  1048. * Similar to split_page except the page is already free. As this is only
  1049. * being used for migration, the migratetype of the block also changes.
  1050. * As this is called with interrupts disabled, the caller is responsible
  1051. * for calling arch_alloc_page() and kernel_map_page() after interrupts
  1052. * are enabled.
  1053. *
  1054. * Note: this is probably too low level an operation for use in drivers.
  1055. * Please consult with lkml before using this in your driver.
  1056. */
  1057. int split_free_page(struct page *page)
  1058. {
  1059. unsigned int order;
  1060. unsigned long watermark;
  1061. struct zone *zone;
  1062. BUG_ON(!PageBuddy(page));
  1063. zone = page_zone(page);
  1064. order = page_order(page);
  1065. /* Obey watermarks as if the page was being allocated */
  1066. watermark = low_wmark_pages(zone) + (1 << order);
  1067. if (!zone_watermark_ok(zone, 0, watermark, 0, 0))
  1068. return 0;
  1069. /* Remove page from free list */
  1070. list_del(&page->lru);
  1071. zone->free_area[order].nr_free--;
  1072. rmv_page_order(page);
  1073. __mod_zone_page_state(zone, NR_FREE_PAGES, -(1UL << order));
  1074. /* Split into individual pages */
  1075. set_page_refcounted(page);
  1076. split_page(page, order);
  1077. if (order >= pageblock_order - 1) {
  1078. struct page *endpage = page + (1 << order) - 1;
  1079. for (; page < endpage; page += pageblock_nr_pages)
  1080. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  1081. }
  1082. return 1 << order;
  1083. }
  1084. /*
  1085. * Really, prep_compound_page() should be called from __rmqueue_bulk(). But
  1086. * we cheat by calling it from here, in the order > 0 path. Saves a branch
  1087. * or two.
  1088. */
  1089. static inline
  1090. struct page *buffered_rmqueue(struct zone *preferred_zone,
  1091. struct zone *zone, int order, gfp_t gfp_flags,
  1092. int migratetype)
  1093. {
  1094. unsigned long flags;
  1095. struct page *page;
  1096. int cold = !!(gfp_flags & __GFP_COLD);
  1097. again:
  1098. if (likely(order == 0)) {
  1099. struct per_cpu_pages *pcp;
  1100. struct list_head *list;
  1101. local_irq_save(flags);
  1102. pcp = &this_cpu_ptr(zone->pageset)->pcp;
  1103. list = &pcp->lists[migratetype];
  1104. if (list_empty(list)) {
  1105. pcp->count += rmqueue_bulk(zone, 0,
  1106. pcp->batch, list,
  1107. migratetype, cold);
  1108. if (unlikely(list_empty(list)))
  1109. goto failed;
  1110. }
  1111. if (cold)
  1112. page = list_entry(list->prev, struct page, lru);
  1113. else
  1114. page = list_entry(list->next, struct page, lru);
  1115. list_del(&page->lru);
  1116. pcp->count--;
  1117. } else {
  1118. if (unlikely(gfp_flags & __GFP_NOFAIL)) {
  1119. /*
  1120. * __GFP_NOFAIL is not to be used in new code.
  1121. *
  1122. * All __GFP_NOFAIL callers should be fixed so that they
  1123. * properly detect and handle allocation failures.
  1124. *
  1125. * We most definitely don't want callers attempting to
  1126. * allocate greater than order-1 page units with
  1127. * __GFP_NOFAIL.
  1128. */
  1129. WARN_ON_ONCE(order > 1);
  1130. }
  1131. spin_lock_irqsave(&zone->lock, flags);
  1132. page = __rmqueue(zone, order, migratetype);
  1133. spin_unlock(&zone->lock);
  1134. if (!page)
  1135. goto failed;
  1136. __mod_zone_page_state(zone, NR_FREE_PAGES, -(1 << order));
  1137. }
  1138. __count_zone_vm_events(PGALLOC, zone, 1 << order);
  1139. zone_statistics(preferred_zone, zone);
  1140. local_irq_restore(flags);
  1141. VM_BUG_ON(bad_range(zone, page));
  1142. if (prep_new_page(page, order, gfp_flags))
  1143. goto again;
  1144. return page;
  1145. failed:
  1146. local_irq_restore(flags);
  1147. return NULL;
  1148. }
  1149. /* The ALLOC_WMARK bits are used as an index to zone->watermark */
  1150. #define ALLOC_WMARK_MIN WMARK_MIN
  1151. #define ALLOC_WMARK_LOW WMARK_LOW
  1152. #define ALLOC_WMARK_HIGH WMARK_HIGH
  1153. #define ALLOC_NO_WATERMARKS 0x04 /* don't check watermarks at all */
  1154. /* Mask to get the watermark bits */
  1155. #define ALLOC_WMARK_MASK (ALLOC_NO_WATERMARKS-1)
  1156. #define ALLOC_HARDER 0x10 /* try to alloc harder */
  1157. #define ALLOC_HIGH 0x20 /* __GFP_HIGH set */
  1158. #define ALLOC_CPUSET 0x40 /* check for correct cpuset */
  1159. #ifdef CONFIG_FAIL_PAGE_ALLOC
  1160. static struct fail_page_alloc_attr {
  1161. struct fault_attr attr;
  1162. u32 ignore_gfp_highmem;
  1163. u32 ignore_gfp_wait;
  1164. u32 min_order;
  1165. #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
  1166. struct dentry *ignore_gfp_highmem_file;
  1167. struct dentry *ignore_gfp_wait_file;
  1168. struct dentry *min_order_file;
  1169. #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
  1170. } fail_page_alloc = {
  1171. .attr = FAULT_ATTR_INITIALIZER,
  1172. .ignore_gfp_wait = 1,
  1173. .ignore_gfp_highmem = 1,
  1174. .min_order = 1,
  1175. };
  1176. static int __init setup_fail_page_alloc(char *str)
  1177. {
  1178. return setup_fault_attr(&fail_page_alloc.attr, str);
  1179. }
  1180. __setup("fail_page_alloc=", setup_fail_page_alloc);
  1181. static int should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
  1182. {
  1183. if (order < fail_page_alloc.min_order)
  1184. return 0;
  1185. if (gfp_mask & __GFP_NOFAIL)
  1186. return 0;
  1187. if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
  1188. return 0;
  1189. if (fail_page_alloc.ignore_gfp_wait && (gfp_mask & __GFP_WAIT))
  1190. return 0;
  1191. return should_fail(&fail_page_alloc.attr, 1 << order);
  1192. }
  1193. #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
  1194. static int __init fail_page_alloc_debugfs(void)
  1195. {
  1196. mode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
  1197. struct dentry *dir;
  1198. int err;
  1199. err = init_fault_attr_dentries(&fail_page_alloc.attr,
  1200. "fail_page_alloc");
  1201. if (err)
  1202. return err;
  1203. dir = fail_page_alloc.attr.dentries.dir;
  1204. fail_page_alloc.ignore_gfp_wait_file =
  1205. debugfs_create_bool("ignore-gfp-wait", mode, dir,
  1206. &fail_page_alloc.ignore_gfp_wait);
  1207. fail_page_alloc.ignore_gfp_highmem_file =
  1208. debugfs_create_bool("ignore-gfp-highmem", mode, dir,
  1209. &fail_page_alloc.ignore_gfp_highmem);
  1210. fail_page_alloc.min_order_file =
  1211. debugfs_create_u32("min-order", mode, dir,
  1212. &fail_page_alloc.min_order);
  1213. if (!fail_page_alloc.ignore_gfp_wait_file ||
  1214. !fail_page_alloc.ignore_gfp_highmem_file ||
  1215. !fail_page_alloc.min_order_file) {
  1216. err = -ENOMEM;
  1217. debugfs_remove(fail_page_alloc.ignore_gfp_wait_file);
  1218. debugfs_remove(fail_page_alloc.ignore_gfp_highmem_file);
  1219. debugfs_remove(fail_page_alloc.min_order_file);
  1220. cleanup_fault_attr_dentries(&fail_page_alloc.attr);
  1221. }
  1222. return err;
  1223. }
  1224. late_initcall(fail_page_alloc_debugfs);
  1225. #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
  1226. #else /* CONFIG_FAIL_PAGE_ALLOC */
  1227. static inline int should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
  1228. {
  1229. return 0;
  1230. }
  1231. #endif /* CONFIG_FAIL_PAGE_ALLOC */
  1232. /*
  1233. * Return 1 if free pages are above 'mark'. This takes into account the order
  1234. * of the allocation.
  1235. */
  1236. int zone_watermark_ok(struct zone *z, int order, unsigned long mark,
  1237. int classzone_idx, int alloc_flags)
  1238. {
  1239. /* free_pages my go negative - that's OK */
  1240. long min = mark;
  1241. long free_pages = zone_page_state(z, NR_FREE_PAGES) - (1 << order) + 1;
  1242. int o;
  1243. if (alloc_flags & ALLOC_HIGH)
  1244. min -= min / 2;
  1245. if (alloc_flags & ALLOC_HARDER)
  1246. min -= min / 4;
  1247. if (free_pages <= min + z->lowmem_reserve[classzone_idx])
  1248. return 0;
  1249. for (o = 0; o < order; o++) {
  1250. /* At the next order, this order's pages become unavailable */
  1251. free_pages -= z->free_area[o].nr_free << o;
  1252. /* Require fewer higher order pages to be free */
  1253. min >>= 1;
  1254. if (free_pages <= min)
  1255. return 0;
  1256. }
  1257. return 1;
  1258. }
  1259. #ifdef CONFIG_NUMA
  1260. /*
  1261. * zlc_setup - Setup for "zonelist cache". Uses cached zone data to
  1262. * skip over zones that are not allowed by the cpuset, or that have
  1263. * been recently (in last second) found to be nearly full. See further
  1264. * comments in mmzone.h. Reduces cache footprint of zonelist scans
  1265. * that have to skip over a lot of full or unallowed zones.
  1266. *
  1267. * If the zonelist cache is present in the passed in zonelist, then
  1268. * returns a pointer to the allowed node mask (either the current
  1269. * tasks mems_allowed, or node_states[N_HIGH_MEMORY].)
  1270. *
  1271. * If the zonelist cache is not available for this zonelist, does
  1272. * nothing and returns NULL.
  1273. *
  1274. * If the fullzones BITMAP in the zonelist cache is stale (more than
  1275. * a second since last zap'd) then we zap it out (clear its bits.)
  1276. *
  1277. * We hold off even calling zlc_setup, until after we've checked the
  1278. * first zone in the zonelist, on the theory that most allocations will
  1279. * be satisfied from that first zone, so best to examine that zone as
  1280. * quickly as we can.
  1281. */
  1282. static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
  1283. {
  1284. struct zonelist_cache *zlc; /* cached zonelist speedup info */
  1285. nodemask_t *allowednodes; /* zonelist_cache approximation */
  1286. zlc = zonelist->zlcache_ptr;
  1287. if (!zlc)
  1288. return NULL;
  1289. if (time_after(jiffies, zlc->last_full_zap + HZ)) {
  1290. bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
  1291. zlc->last_full_zap = jiffies;
  1292. }
  1293. allowednodes = !in_interrupt() && (alloc_flags & ALLOC_CPUSET) ?
  1294. &cpuset_current_mems_allowed :
  1295. &node_states[N_HIGH_MEMORY];
  1296. return allowednodes;
  1297. }
  1298. /*
  1299. * Given 'z' scanning a zonelist, run a couple of quick checks to see
  1300. * if it is worth looking at further for free memory:
  1301. * 1) Check that the zone isn't thought to be full (doesn't have its
  1302. * bit set in the zonelist_cache fullzones BITMAP).
  1303. * 2) Check that the zones node (obtained from the zonelist_cache
  1304. * z_to_n[] mapping) is allowed in the passed in allowednodes mask.
  1305. * Return true (non-zero) if zone is worth looking at further, or
  1306. * else return false (zero) if it is not.
  1307. *
  1308. * This check -ignores- the distinction between various watermarks,
  1309. * such as GFP_HIGH, GFP_ATOMIC, PF_MEMALLOC, ... If a zone is
  1310. * found to be full for any variation of these watermarks, it will
  1311. * be considered full for up to one second by all requests, unless
  1312. * we are so low on memory on all allowed nodes that we are forced
  1313. * into the second scan of the zonelist.
  1314. *
  1315. * In the second scan we ignore this zonelist cache and exactly
  1316. * apply the watermarks to all zones, even it is slower to do so.
  1317. * We are low on memory in the second scan, and should leave no stone
  1318. * unturned looking for a free page.
  1319. */
  1320. static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z,
  1321. nodemask_t *allowednodes)
  1322. {
  1323. struct zonelist_cache *zlc; /* cached zonelist speedup info */
  1324. int i; /* index of *z in zonelist zones */
  1325. int n; /* node that zone *z is on */
  1326. zlc = zonelist->zlcache_ptr;
  1327. if (!zlc)
  1328. return 1;
  1329. i = z - zonelist->_zonerefs;
  1330. n = zlc->z_to_n[i];
  1331. /* This zone is worth trying if it is allowed but not full */
  1332. return node_isset(n, *allowednodes) && !test_bit(i, zlc->fullzones);
  1333. }
  1334. /*
  1335. * Given 'z' scanning a zonelist, set the corresponding bit in
  1336. * zlc->fullzones, so that subsequent attempts to allocate a page
  1337. * from that zone don't waste time re-examining it.
  1338. */
  1339. static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z)
  1340. {
  1341. struct zonelist_cache *zlc; /* cached zonelist speedup info */
  1342. int i; /* index of *z in zonelist zones */
  1343. zlc = zonelist->zlcache_ptr;
  1344. if (!zlc)
  1345. return;
  1346. i = z - zonelist->_zonerefs;
  1347. set_bit(i, zlc->fullzones);
  1348. }
  1349. #else /* CONFIG_NUMA */
  1350. static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
  1351. {
  1352. return NULL;
  1353. }
  1354. static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z,
  1355. nodemask_t *allowednodes)
  1356. {
  1357. return 1;
  1358. }
  1359. static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z)
  1360. {
  1361. }
  1362. #endif /* CONFIG_NUMA */
  1363. /*
  1364. * get_page_from_freelist goes through the zonelist trying to allocate
  1365. * a page.
  1366. */
  1367. static struct page *
  1368. get_page_from_freelist(gfp_t gfp_mask, nodemask_t *nodemask, unsigned int order,
  1369. struct zonelist *zonelist, int high_zoneidx, int alloc_flags,
  1370. struct zone *preferred_zone, int migratetype)
  1371. {
  1372. struct zoneref *z;
  1373. struct page *page = NULL;
  1374. int classzone_idx;
  1375. struct zone *zone;
  1376. nodemask_t *allowednodes = NULL;/* zonelist_cache approximation */
  1377. int zlc_active = 0; /* set if using zonelist_cache */
  1378. int did_zlc_setup = 0; /* just call zlc_setup() one time */
  1379. classzone_idx = zone_idx(preferred_zone);
  1380. zonelist_scan:
  1381. /*
  1382. * Scan zonelist, looking for a zone with enough free.
  1383. * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
  1384. */
  1385. for_each_zone_zonelist_nodemask(zone, z, zonelist,
  1386. high_zoneidx, nodemask) {
  1387. if (NUMA_BUILD && zlc_active &&
  1388. !zlc_zone_worth_trying(zonelist, z, allowednodes))
  1389. continue;
  1390. if ((alloc_flags & ALLOC_CPUSET) &&
  1391. !cpuset_zone_allowed_softwall(zone, gfp_mask))
  1392. goto try_next_zone;
  1393. BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
  1394. if (!(alloc_flags & ALLOC_NO_WATERMARKS)) {
  1395. unsigned long mark;
  1396. int ret;
  1397. mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK];
  1398. if (zone_watermark_ok(zone, order, mark,
  1399. classzone_idx, alloc_flags))
  1400. goto try_this_zone;
  1401. if (zone_reclaim_mode == 0)
  1402. goto this_zone_full;
  1403. ret = zone_reclaim(zone, gfp_mask, order);
  1404. switch (ret) {
  1405. case ZONE_RECLAIM_NOSCAN:
  1406. /* did not scan */
  1407. goto try_next_zone;
  1408. case ZONE_RECLAIM_FULL:
  1409. /* scanned but unreclaimable */
  1410. goto this_zone_full;
  1411. default:
  1412. /* did we reclaim enough */
  1413. if (!zone_watermark_ok(zone, order, mark,
  1414. classzone_idx, alloc_flags))
  1415. goto this_zone_full;
  1416. }
  1417. }
  1418. try_this_zone:
  1419. page = buffered_rmqueue(preferred_zone, zone, order,
  1420. gfp_mask, migratetype);
  1421. if (page)
  1422. break;
  1423. this_zone_full:
  1424. if (NUMA_BUILD)
  1425. zlc_mark_zone_full(zonelist, z);
  1426. try_next_zone:
  1427. if (NUMA_BUILD && !did_zlc_setup && nr_online_nodes > 1) {
  1428. /*
  1429. * we do zlc_setup after the first zone is tried but only
  1430. * if there are multiple nodes make it worthwhile
  1431. */
  1432. allowednodes = zlc_setup(zonelist, alloc_flags);
  1433. zlc_active = 1;
  1434. did_zlc_setup = 1;
  1435. }
  1436. }
  1437. if (unlikely(NUMA_BUILD && page == NULL && zlc_active)) {
  1438. /* Disable zlc cache for second zonelist scan */
  1439. zlc_active = 0;
  1440. goto zonelist_scan;
  1441. }
  1442. return page;
  1443. }
  1444. static inline int
  1445. should_alloc_retry(gfp_t gfp_mask, unsigned int order,
  1446. unsigned long pages_reclaimed)
  1447. {
  1448. /* Do not loop if specifically requested */
  1449. if (gfp_mask & __GFP_NORETRY)
  1450. return 0;
  1451. /*
  1452. * In this implementation, order <= PAGE_ALLOC_COSTLY_ORDER
  1453. * means __GFP_NOFAIL, but that may not be true in other
  1454. * implementations.
  1455. */
  1456. if (order <= PAGE_ALLOC_COSTLY_ORDER)
  1457. return 1;
  1458. /*
  1459. * For order > PAGE_ALLOC_COSTLY_ORDER, if __GFP_REPEAT is
  1460. * specified, then we retry until we no longer reclaim any pages
  1461. * (above), or we've reclaimed an order of pages at least as
  1462. * large as the allocation's order. In both cases, if the
  1463. * allocation still fails, we stop retrying.
  1464. */
  1465. if (gfp_mask & __GFP_REPEAT && pages_reclaimed < (1 << order))
  1466. return 1;
  1467. /*
  1468. * Don't let big-order allocations loop unless the caller
  1469. * explicitly requests that.
  1470. */
  1471. if (gfp_mask & __GFP_NOFAIL)
  1472. return 1;
  1473. return 0;
  1474. }
  1475. static inline struct page *
  1476. __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
  1477. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1478. nodemask_t *nodemask, struct zone *preferred_zone,
  1479. int migratetype)
  1480. {
  1481. struct page *page;
  1482. /* Acquire the OOM killer lock for the zones in zonelist */
  1483. if (!try_set_zone_oom(zonelist, gfp_mask)) {
  1484. schedule_timeout_uninterruptible(1);
  1485. return NULL;
  1486. }
  1487. /*
  1488. * Go through the zonelist yet one more time, keep very high watermark
  1489. * here, this is only to catch a parallel oom killing, we must fail if
  1490. * we're still under heavy pressure.
  1491. */
  1492. page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask,
  1493. order, zonelist, high_zoneidx,
  1494. ALLOC_WMARK_HIGH|ALLOC_CPUSET,
  1495. preferred_zone, migratetype);
  1496. if (page)
  1497. goto out;
  1498. if (!(gfp_mask & __GFP_NOFAIL)) {
  1499. /* The OOM killer will not help higher order allocs */
  1500. if (order > PAGE_ALLOC_COSTLY_ORDER)
  1501. goto out;
  1502. /*
  1503. * GFP_THISNODE contains __GFP_NORETRY and we never hit this.
  1504. * Sanity check for bare calls of __GFP_THISNODE, not real OOM.
  1505. * The caller should handle page allocation failure by itself if
  1506. * it specifies __GFP_THISNODE.
  1507. * Note: Hugepage uses it but will hit PAGE_ALLOC_COSTLY_ORDER.
  1508. */
  1509. if (gfp_mask & __GFP_THISNODE)
  1510. goto out;
  1511. }
  1512. /* Exhausted what can be done so it's blamo time */
  1513. out_of_memory(zonelist, gfp_mask, order, nodemask);
  1514. out:
  1515. clear_zonelist_oom(zonelist, gfp_mask);
  1516. return page;
  1517. }
  1518. #ifdef CONFIG_COMPACTION
  1519. /* Try memory compaction for high-order allocations before reclaim */
  1520. static struct page *
  1521. __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
  1522. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1523. nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
  1524. int migratetype, unsigned long *did_some_progress)
  1525. {
  1526. struct page *page;
  1527. if (!order || compaction_deferred(preferred_zone))
  1528. return NULL;
  1529. *did_some_progress = try_to_compact_pages(zonelist, order, gfp_mask,
  1530. nodemask);
  1531. if (*did_some_progress != COMPACT_SKIPPED) {
  1532. /* Page migration frees to the PCP lists but we want merging */
  1533. drain_pages(get_cpu());
  1534. put_cpu();
  1535. page = get_page_from_freelist(gfp_mask, nodemask,
  1536. order, zonelist, high_zoneidx,
  1537. alloc_flags, preferred_zone,
  1538. migratetype);
  1539. if (page) {
  1540. preferred_zone->compact_considered = 0;
  1541. preferred_zone->compact_defer_shift = 0;
  1542. count_vm_event(COMPACTSUCCESS);
  1543. return page;
  1544. }
  1545. /*
  1546. * It's bad if compaction run occurs and fails.
  1547. * The most likely reason is that pages exist,
  1548. * but not enough to satisfy watermarks.
  1549. */
  1550. count_vm_event(COMPACTFAIL);
  1551. defer_compaction(preferred_zone);
  1552. cond_resched();
  1553. }
  1554. return NULL;
  1555. }
  1556. #else
  1557. static inline struct page *
  1558. __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
  1559. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1560. nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
  1561. int migratetype, unsigned long *did_some_progress)
  1562. {
  1563. return NULL;
  1564. }
  1565. #endif /* CONFIG_COMPACTION */
  1566. /* The really slow allocator path where we enter direct reclaim */
  1567. static inline struct page *
  1568. __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
  1569. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1570. nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
  1571. int migratetype, unsigned long *did_some_progress)
  1572. {
  1573. struct page *page = NULL;
  1574. struct reclaim_state reclaim_state;
  1575. struct task_struct *p = current;
  1576. cond_resched();
  1577. /* We now go into synchronous reclaim */
  1578. cpuset_memory_pressure_bump();
  1579. p->flags |= PF_MEMALLOC;
  1580. lockdep_set_current_reclaim_state(gfp_mask);
  1581. reclaim_state.reclaimed_slab = 0;
  1582. p->reclaim_state = &reclaim_state;
  1583. *did_some_progress = try_to_free_pages(zonelist, order, gfp_mask, nodemask);
  1584. p->reclaim_state = NULL;
  1585. lockdep_clear_current_reclaim_state();
  1586. p->flags &= ~PF_MEMALLOC;
  1587. cond_resched();
  1588. if (order != 0)
  1589. drain_all_pages();
  1590. if (likely(*did_some_progress))
  1591. page = get_page_from_freelist(gfp_mask, nodemask, order,
  1592. zonelist, high_zoneidx,
  1593. alloc_flags, preferred_zone,
  1594. migratetype);
  1595. return page;
  1596. }
  1597. /*
  1598. * This is called in the allocator slow-path if the allocation request is of
  1599. * sufficient urgency to ignore watermarks and take other desperate measures
  1600. */
  1601. static inline struct page *
  1602. __alloc_pages_high_priority(gfp_t gfp_mask, unsigned int order,
  1603. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1604. nodemask_t *nodemask, struct zone *preferred_zone,
  1605. int migratetype)
  1606. {
  1607. struct page *page;
  1608. do {
  1609. page = get_page_from_freelist(gfp_mask, nodemask, order,
  1610. zonelist, high_zoneidx, ALLOC_NO_WATERMARKS,
  1611. preferred_zone, migratetype);
  1612. if (!page && gfp_mask & __GFP_NOFAIL)
  1613. congestion_wait(BLK_RW_ASYNC, HZ/50);
  1614. } while (!page && (gfp_mask & __GFP_NOFAIL));
  1615. return page;
  1616. }
  1617. static inline
  1618. void wake_all_kswapd(unsigned int order, struct zonelist *zonelist,
  1619. enum zone_type high_zoneidx)
  1620. {
  1621. struct zoneref *z;
  1622. struct zone *zone;
  1623. for_each_zone_zonelist(zone, z, zonelist, high_zoneidx)
  1624. wakeup_kswapd(zone, order);
  1625. }
  1626. static inline int
  1627. gfp_to_alloc_flags(gfp_t gfp_mask)
  1628. {
  1629. struct task_struct *p = current;
  1630. int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
  1631. const gfp_t wait = gfp_mask & __GFP_WAIT;
  1632. /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */
  1633. BUILD_BUG_ON(__GFP_HIGH != ALLOC_HIGH);
  1634. /*
  1635. * The caller may dip into page reserves a bit more if the caller
  1636. * cannot run direct reclaim, or if the caller has realtime scheduling
  1637. * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will
  1638. * set both ALLOC_HARDER (!wait) and ALLOC_HIGH (__GFP_HIGH).
  1639. */
  1640. alloc_flags |= (gfp_mask & __GFP_HIGH);
  1641. if (!wait) {
  1642. alloc_flags |= ALLOC_HARDER;
  1643. /*
  1644. * Ignore cpuset if GFP_ATOMIC (!wait) rather than fail alloc.
  1645. * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
  1646. */
  1647. alloc_flags &= ~ALLOC_CPUSET;
  1648. } else if (unlikely(rt_task(p)) && !in_interrupt())
  1649. alloc_flags |= ALLOC_HARDER;
  1650. if (likely(!(gfp_mask & __GFP_NOMEMALLOC))) {
  1651. if (!in_interrupt() &&
  1652. ((p->flags & PF_MEMALLOC) ||
  1653. unlikely(test_thread_flag(TIF_MEMDIE))))
  1654. alloc_flags |= ALLOC_NO_WATERMARKS;
  1655. }
  1656. return alloc_flags;
  1657. }
  1658. static inline struct page *
  1659. __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
  1660. struct zonelist *zonelist, enum zone_type high_zoneidx,
  1661. nodemask_t *nodemask, struct zone *preferred_zone,
  1662. int migratetype)
  1663. {
  1664. const gfp_t wait = gfp_mask & __GFP_WAIT;
  1665. struct page *page = NULL;
  1666. int alloc_flags;
  1667. unsigned long pages_reclaimed = 0;
  1668. unsigned long did_some_progress;
  1669. struct task_struct *p = current;
  1670. /*
  1671. * In the slowpath, we sanity check order to avoid ever trying to
  1672. * reclaim >= MAX_ORDER areas which will never succeed. Callers may
  1673. * be using allocators in order of preference for an area that is
  1674. * too large.
  1675. */
  1676. if (order >= MAX_ORDER) {
  1677. WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));
  1678. return NULL;
  1679. }
  1680. /*
  1681. * GFP_THISNODE (meaning __GFP_THISNODE, __GFP_NORETRY and
  1682. * __GFP_NOWARN set) should not cause reclaim since the subsystem
  1683. * (f.e. slab) using GFP_THISNODE may choose to trigger reclaim
  1684. * using a larger set of nodes after it has established that the
  1685. * allowed per node queues are empty and that nodes are
  1686. * over allocated.
  1687. */
  1688. if (NUMA_BUILD && (gfp_mask & GFP_THISNODE) == GFP_THISNODE)
  1689. goto nopage;
  1690. restart:
  1691. wake_all_kswapd(order, zonelist, high_zoneidx);
  1692. /*
  1693. * OK, we're below the kswapd watermark and have kicked background
  1694. * reclaim. Now things get more complex, so set up alloc_flags according
  1695. * to how we want to proceed.
  1696. */
  1697. alloc_flags = gfp_to_alloc_flags(gfp_mask);
  1698. /* This is the last chance, in general, before the goto nopage. */
  1699. page = get_page_from_freelist(gfp_mask, nodemask, order, zonelist,
  1700. high_zoneidx, alloc_flags & ~ALLOC_NO_WATERMARKS,
  1701. preferred_zone, migratetype);
  1702. if (page)
  1703. goto got_pg;
  1704. rebalance:
  1705. /* Allocate without watermarks if the context allows */
  1706. if (alloc_flags & ALLOC_NO_WATERMARKS) {
  1707. page = __alloc_pages_high_priority(gfp_mask, order,
  1708. zonelist, high_zoneidx, nodemask,
  1709. preferred_zone, migratetype);
  1710. if (page)
  1711. goto got_pg;
  1712. }
  1713. /* Atomic allocations - we can't balance anything */
  1714. if (!wait)
  1715. goto nopage;
  1716. /* Avoid recursion of direct reclaim */
  1717. if (p->flags & PF_MEMALLOC)
  1718. goto nopage;
  1719. /* Avoid allocations with no watermarks from looping endlessly */
  1720. if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL))
  1721. goto nopage;
  1722. /* Try direct compaction */
  1723. page = __alloc_pages_direct_compact(gfp_mask, order,
  1724. zonelist, high_zoneidx,
  1725. nodemask,
  1726. alloc_flags, preferred_zone,
  1727. migratetype, &did_some_progress);
  1728. if (page)
  1729. goto got_pg;
  1730. /* Try direct reclaim and then allocating */
  1731. page = __alloc_pages_direct_reclaim(gfp_mask, order,
  1732. zonelist, high_zoneidx,
  1733. nodemask,
  1734. alloc_flags, preferred_zone,
  1735. migratetype, &did_some_progress);
  1736. if (page)
  1737. goto got_pg;
  1738. /*
  1739. * If we failed to make any progress reclaiming, then we are
  1740. * running out of options and have to consider going OOM
  1741. */
  1742. if (!did_some_progress) {
  1743. if ((gfp_mask & __GFP_FS) && !(gfp_mask & __GFP_NORETRY)) {
  1744. if (oom_killer_disabled)
  1745. goto nopage;
  1746. page = __alloc_pages_may_oom(gfp_mask, order,
  1747. zonelist, high_zoneidx,
  1748. nodemask, preferred_zone,
  1749. migratetype);
  1750. if (page)
  1751. goto got_pg;
  1752. /*
  1753. * The OOM killer does not trigger for high-order
  1754. * ~__GFP_NOFAIL allocations so if no progress is being
  1755. * made, there are no other options and retrying is
  1756. * unlikely to help.
  1757. */
  1758. if (order > PAGE_ALLOC_COSTLY_ORDER &&
  1759. !(gfp_mask & __GFP_NOFAIL))
  1760. goto nopage;
  1761. goto restart;
  1762. }
  1763. }
  1764. /* Check if we should retry the allocation */
  1765. pages_reclaimed += did_some_progress;
  1766. if (should_alloc_retry(gfp_mask, order, pages_reclaimed)) {
  1767. /* Wait for some write requests to complete then retry */
  1768. congestion_wait(BLK_RW_ASYNC, HZ/50);
  1769. goto rebalance;
  1770. }
  1771. nopage:
  1772. if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit()) {
  1773. printk(KERN_WARNING "%s: page allocation failure."
  1774. " order:%d, mode:0x%x\n",
  1775. p->comm, order, gfp_mask);
  1776. dump_stack();
  1777. show_mem();
  1778. }
  1779. return page;
  1780. got_pg:
  1781. if (kmemcheck_enabled)
  1782. kmemcheck_pagealloc_alloc(page, order, gfp_mask);
  1783. return page;
  1784. }
  1785. /*
  1786. * This is the 'heart' of the zoned buddy allocator.
  1787. */
  1788. struct page *
  1789. __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
  1790. struct zonelist *zonelist, nodemask_t *nodemask)
  1791. {
  1792. enum zone_type high_zoneidx = gfp_zone(gfp_mask);
  1793. struct zone *preferred_zone;
  1794. struct page *page;
  1795. int migratetype = allocflags_to_migratetype(gfp_mask);
  1796. gfp_mask &= gfp_allowed_mask;
  1797. lockdep_trace_alloc(gfp_mask);
  1798. might_sleep_if(gfp_mask & __GFP_WAIT);
  1799. if (should_fail_alloc_page(gfp_mask, order))
  1800. return NULL;
  1801. /*
  1802. * Check the zones suitable for the gfp_mask contain at least one
  1803. * valid zone. It's possible to have an empty zonelist as a result
  1804. * of GFP_THISNODE and a memoryless node
  1805. */
  1806. if (unlikely(!zonelist->_zonerefs->zone))
  1807. return NULL;
  1808. get_mems_allowed();
  1809. /* The preferred zone is used for statistics later */
  1810. first_zones_zonelist(zonelist, high_zoneidx, nodemask, &preferred_zone);
  1811. if (!preferred_zone) {
  1812. put_mems_allowed();
  1813. return NULL;
  1814. }
  1815. /* First allocation attempt */
  1816. page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask, order,
  1817. zonelist, high_zoneidx, ALLOC_WMARK_LOW|ALLOC_CPUSET,
  1818. preferred_zone, migratetype);
  1819. if (unlikely(!page))
  1820. page = __alloc_pages_slowpath(gfp_mask, order,
  1821. zonelist, high_zoneidx, nodemask,
  1822. preferred_zone, migratetype);
  1823. put_mems_allowed();
  1824. trace_mm_page_alloc(page, order, gfp_mask, migratetype);
  1825. return page;
  1826. }
  1827. EXPORT_SYMBOL(__alloc_pages_nodemask);
  1828. /*
  1829. * Common helper functions.
  1830. */
  1831. unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
  1832. {
  1833. struct page *page;
  1834. /*
  1835. * __get_free_pages() returns a 32-bit address, which cannot represent
  1836. * a highmem page
  1837. */
  1838. VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0);
  1839. page = alloc_pages(gfp_mask, order);
  1840. if (!page)
  1841. return 0;
  1842. return (unsigned long) page_address(page);
  1843. }
  1844. EXPORT_SYMBOL(__get_free_pages);
  1845. unsigned long get_zeroed_page(gfp_t gfp_mask)
  1846. {
  1847. return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
  1848. }
  1849. EXPORT_SYMBOL(get_zeroed_page);
  1850. void __pagevec_free(struct pagevec *pvec)
  1851. {
  1852. int i = pagevec_count(pvec);
  1853. while (--i >= 0) {
  1854. trace_mm_pagevec_free(pvec->pages[i], pvec->cold);
  1855. free_hot_cold_page(pvec->pages[i], pvec->cold);
  1856. }
  1857. }
  1858. void __free_pages(struct page *page, unsigned int order)
  1859. {
  1860. if (put_page_testzero(page)) {
  1861. if (order == 0)
  1862. free_hot_cold_page(page, 0);
  1863. else
  1864. __free_pages_ok(page, order);
  1865. }
  1866. }
  1867. EXPORT_SYMBOL(__free_pages);
  1868. void free_pages(unsigned long addr, unsigned int order)
  1869. {
  1870. if (addr != 0) {
  1871. VM_BUG_ON(!virt_addr_valid((void *)addr));
  1872. __free_pages(virt_to_page((void *)addr), order);
  1873. }
  1874. }
  1875. EXPORT_SYMBOL(free_pages);
  1876. /**
  1877. * alloc_pages_exact - allocate an exact number physically-contiguous pages.
  1878. * @size: the number of bytes to allocate
  1879. * @gfp_mask: GFP flags for the allocation
  1880. *
  1881. * This function is similar to alloc_pages(), except that it allocates the
  1882. * minimum number of pages to satisfy the request. alloc_pages() can only
  1883. * allocate memory in power-of-two pages.
  1884. *
  1885. * This function is also limited by MAX_ORDER.
  1886. *
  1887. * Memory allocated by this function must be released by free_pages_exact().
  1888. */
  1889. void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
  1890. {
  1891. unsigned int order = get_order(size);
  1892. unsigned long addr;
  1893. addr = __get_free_pages(gfp_mask, order);
  1894. if (addr) {
  1895. unsigned long alloc_end = addr + (PAGE_SIZE << order);
  1896. unsigned long used = addr + PAGE_ALIGN(size);
  1897. split_page(virt_to_page((void *)addr), order);
  1898. while (used < alloc_end) {
  1899. free_page(used);
  1900. used += PAGE_SIZE;
  1901. }
  1902. }
  1903. return (void *)addr;
  1904. }
  1905. EXPORT_SYMBOL(alloc_pages_exact);
  1906. /**
  1907. * free_pages_exact - release memory allocated via alloc_pages_exact()
  1908. * @virt: the value returned by alloc_pages_exact.
  1909. * @size: size of allocation, same value as passed to alloc_pages_exact().
  1910. *
  1911. * Release the memory allocated by a previous call to alloc_pages_exact.
  1912. */
  1913. void free_pages_exact(void *virt, size_t size)
  1914. {
  1915. unsigned long addr = (unsigned long)virt;
  1916. unsigned long end = addr + PAGE_ALIGN(size);
  1917. while (addr < end) {
  1918. free_page(addr);
  1919. addr += PAGE_SIZE;
  1920. }
  1921. }
  1922. EXPORT_SYMBOL(free_pages_exact);
  1923. static unsigned int nr_free_zone_pages(int offset)
  1924. {
  1925. struct zoneref *z;
  1926. struct zone *zone;
  1927. /* Just pick one node, since fallback list is circular */
  1928. unsigned int sum = 0;
  1929. struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
  1930. for_each_zone_zonelist(zone, z, zonelist, offset) {
  1931. unsigned long size = zone->present_pages;
  1932. unsigned long high = high_wmark_pages(zone);
  1933. if (size > high)
  1934. sum += size - high;
  1935. }
  1936. return sum;
  1937. }
  1938. /*
  1939. * Amount of free RAM allocatable within ZONE_DMA and ZONE_NORMAL
  1940. */
  1941. unsigned int nr_free_buffer_pages(void)
  1942. {
  1943. return nr_free_zone_pages(gfp_zone(GFP_USER));
  1944. }
  1945. EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
  1946. /*
  1947. * Amount of free RAM allocatable within all zones
  1948. */
  1949. unsigned int nr_free_pagecache_pages(void)
  1950. {
  1951. return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
  1952. }
  1953. static inline void show_node(struct zone *zone)
  1954. {
  1955. if (NUMA_BUILD)
  1956. printk("Node %d ", zone_to_nid(zone));
  1957. }
  1958. void si_meminfo(struct sysinfo *val)
  1959. {
  1960. val->totalram = totalram_pages;
  1961. val->sharedram = 0;
  1962. val->freeram = global_page_state(NR_FREE_PAGES);
  1963. val->bufferram = nr_blockdev_pages();
  1964. val->totalhigh = totalhigh_pages;
  1965. val->freehigh = nr_free_highpages();
  1966. val->mem_unit = PAGE_SIZE;
  1967. }
  1968. EXPORT_SYMBOL(si_meminfo);
  1969. #ifdef CONFIG_NUMA
  1970. void si_meminfo_node(struct sysinfo *val, int nid)
  1971. {
  1972. pg_data_t *pgdat = NODE_DATA(nid);
  1973. val->totalram = pgdat->node_present_pages;
  1974. val->freeram = node_page_state(nid, NR_FREE_PAGES);
  1975. #ifdef CONFIG_HIGHMEM
  1976. val->totalhigh = pgdat->node_zones[ZONE_HIGHMEM].present_pages;
  1977. val->freehigh = zone_page_state(&pgdat->node_zones[ZONE_HIGHMEM],
  1978. NR_FREE_PAGES);
  1979. #else
  1980. val->totalhigh = 0;
  1981. val->freehigh = 0;
  1982. #endif
  1983. val->mem_unit = PAGE_SIZE;
  1984. }
  1985. #endif
  1986. #define K(x) ((x) << (PAGE_SHIFT-10))
  1987. /*
  1988. * Show free area list (used inside shift_scroll-lock stuff)
  1989. * We also calculate the percentage fragmentation. We do this by counting the
  1990. * memory on each free list with the exception of the first item on the list.
  1991. */
  1992. void show_free_areas(void)
  1993. {
  1994. int cpu;
  1995. struct zone *zone;
  1996. for_each_populated_zone(zone) {
  1997. show_node(zone);
  1998. printk("%s per-cpu:\n", zone->name);
  1999. for_each_online_cpu(cpu) {
  2000. struct per_cpu_pageset *pageset;
  2001. pageset = per_cpu_ptr(zone->pageset, cpu);
  2002. printk("CPU %4d: hi:%5d, btch:%4d usd:%4d\n",
  2003. cpu, pageset->pcp.high,
  2004. pageset->pcp.batch, pageset->pcp.count);
  2005. }
  2006. }
  2007. printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n"
  2008. " active_file:%lu inactive_file:%lu isolated_file:%lu\n"
  2009. " unevictable:%lu"
  2010. " dirty:%lu writeback:%lu unstable:%lu\n"
  2011. " free:%lu slab_reclaimable:%lu slab_unreclaimable:%lu\n"
  2012. " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n",
  2013. global_page_state(NR_ACTIVE_ANON),
  2014. global_page_state(NR_INACTIVE_ANON),
  2015. global_page_state(NR_ISOLATED_ANON),
  2016. global_page_state(NR_ACTIVE_FILE),
  2017. global_page_state(NR_INACTIVE_FILE),
  2018. global_page_state(NR_ISOLATED_FILE),
  2019. global_page_state(NR_UNEVICTABLE),
  2020. global_page_state(NR_FILE_DIRTY),
  2021. global_page_state(NR_WRITEBACK),
  2022. global_page_state(NR_UNSTABLE_NFS),
  2023. global_page_state(NR_FREE_PAGES),
  2024. global_page_state(NR_SLAB_RECLAIMABLE),
  2025. global_page_state(NR_SLAB_UNRECLAIMABLE),
  2026. global_page_state(NR_FILE_MAPPED),
  2027. global_page_state(NR_SHMEM),
  2028. global_page_state(NR_PAGETABLE),
  2029. global_page_state(NR_BOUNCE));
  2030. for_each_populated_zone(zone) {
  2031. int i;
  2032. show_node(zone);
  2033. printk("%s"
  2034. " free:%lukB"
  2035. " min:%lukB"
  2036. " low:%lukB"
  2037. " high:%lukB"
  2038. " active_anon:%lukB"
  2039. " inactive_anon:%lukB"
  2040. " active_file:%lukB"
  2041. " inactive_file:%lukB"
  2042. " unevictable:%lukB"
  2043. " isolated(anon):%lukB"
  2044. " isolated(file):%lukB"
  2045. " present:%lukB"
  2046. " mlocked:%lukB"
  2047. " dirty:%lukB"
  2048. " writeback:%lukB"
  2049. " mapped:%lukB"
  2050. " shmem:%lukB"
  2051. " slab_reclaimable:%lukB"
  2052. " slab_unreclaimable:%lukB"
  2053. " kernel_stack:%lukB"
  2054. " pagetables:%lukB"
  2055. " unstable:%lukB"
  2056. " bounce:%lukB"
  2057. " writeback_tmp:%lukB"
  2058. " pages_scanned:%lu"
  2059. " all_unreclaimable? %s"
  2060. "\n",
  2061. zone->name,
  2062. K(zone_page_state(zone, NR_FREE_PAGES)),
  2063. K(min_wmark_pages(zone)),
  2064. K(low_wmark_pages(zone)),
  2065. K(high_wmark_pages(zone)),
  2066. K(zone_page_state(zone, NR_ACTIVE_ANON)),
  2067. K(zone_page_state(zone, NR_INACTIVE_ANON)),
  2068. K(zone_page_state(zone, NR_ACTIVE_FILE)),
  2069. K(zone_page_state(zone, NR_INACTIVE_FILE)),
  2070. K(zone_page_state(zone, NR_UNEVICTABLE)),
  2071. K(zone_page_state(zone, NR_ISOLATED_ANON)),
  2072. K(zone_page_state(zone, NR_ISOLATED_FILE)),
  2073. K(zone->present_pages),
  2074. K(zone_page_state(zone, NR_MLOCK)),
  2075. K(zone_page_state(zone, NR_FILE_DIRTY)),
  2076. K(zone_page_state(zone, NR_WRITEBACK)),
  2077. K(zone_page_state(zone, NR_FILE_MAPPED)),
  2078. K(zone_page_state(zone, NR_SHMEM)),
  2079. K(zone_page_state(zone, NR_SLAB_RECLAIMABLE)),
  2080. K(zone_page_state(zone, NR_SLAB_UNRECLAIMABLE)),
  2081. zone_page_state(zone, NR_KERNEL_STACK) *
  2082. THREAD_SIZE / 1024,
  2083. K(zone_page_state(zone, NR_PAGETABLE)),
  2084. K(zone_page_state(zone, NR_UNSTABLE_NFS)),
  2085. K(zone_page_state(zone, NR_BOUNCE)),
  2086. K(zone_page_state(zone, NR_WRITEBACK_TEMP)),
  2087. zone->pages_scanned,
  2088. (zone->all_unreclaimable ? "yes" : "no")
  2089. );
  2090. printk("lowmem_reserve[]:");
  2091. for (i = 0; i < MAX_NR_ZONES; i++)
  2092. printk(" %lu", zone->lowmem_reserve[i]);
  2093. printk("\n");
  2094. }
  2095. for_each_populated_zone(zone) {
  2096. unsigned long nr[MAX_ORDER], flags, order, total = 0;
  2097. show_node(zone);
  2098. printk("%s: ", zone->name);
  2099. spin_lock_irqsave(&zone->lock, flags);
  2100. for (order = 0; order < MAX_ORDER; order++) {
  2101. nr[order] = zone->free_area[order].nr_free;
  2102. total += nr[order] << order;
  2103. }
  2104. spin_unlock_irqrestore(&zone->lock, flags);
  2105. for (order = 0; order < MAX_ORDER; order++)
  2106. printk("%lu*%lukB ", nr[order], K(1UL) << order);
  2107. printk("= %lukB\n", K(total));
  2108. }
  2109. printk("%ld total pagecache pages\n", global_page_state(NR_FILE_PAGES));
  2110. show_swap_cache_info();
  2111. }
  2112. static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
  2113. {
  2114. zoneref->zone = zone;
  2115. zoneref->zone_idx = zone_idx(zone);
  2116. }
  2117. /*
  2118. * Builds allocation fallback zone lists.
  2119. *
  2120. * Add all populated zones of a node to the zonelist.
  2121. */
  2122. static int build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist,
  2123. int nr_zones, enum zone_type zone_type)
  2124. {
  2125. struct zone *zone;
  2126. BUG_ON(zone_type >= MAX_NR_ZONES);
  2127. zone_type++;
  2128. do {
  2129. zone_type--;
  2130. zone = pgdat->node_zones + zone_type;
  2131. if (populated_zone(zone)) {
  2132. zoneref_set_zone(zone,
  2133. &zonelist->_zonerefs[nr_zones++]);
  2134. check_highest_zone(zone_type);
  2135. }
  2136. } while (zone_type);
  2137. return nr_zones;
  2138. }
  2139. /*
  2140. * zonelist_order:
  2141. * 0 = automatic detection of better ordering.
  2142. * 1 = order by ([node] distance, -zonetype)
  2143. * 2 = order by (-zonetype, [node] distance)
  2144. *
  2145. * If not NUMA, ZONELIST_ORDER_ZONE and ZONELIST_ORDER_NODE will create
  2146. * the same zonelist. So only NUMA can configure this param.
  2147. */
  2148. #define ZONELIST_ORDER_DEFAULT 0
  2149. #define ZONELIST_ORDER_NODE 1
  2150. #define ZONELIST_ORDER_ZONE 2
  2151. /* zonelist order in the kernel.
  2152. * set_zonelist_order() will set this to NODE or ZONE.
  2153. */
  2154. static int current_zonelist_order = ZONELIST_ORDER_DEFAULT;
  2155. static char zonelist_order_name[3][8] = {"Default", "Node", "Zone"};
  2156. #ifdef CONFIG_NUMA
  2157. /* The value user specified ....changed by config */
  2158. static int user_zonelist_order = ZONELIST_ORDER_DEFAULT;
  2159. /* string for sysctl */
  2160. #define NUMA_ZONELIST_ORDER_LEN 16
  2161. char numa_zonelist_order[16] = "default";
  2162. /*
  2163. * interface for configure zonelist ordering.
  2164. * command line option "numa_zonelist_order"
  2165. * = "[dD]efault - default, automatic configuration.
  2166. * = "[nN]ode - order by node locality, then by zone within node
  2167. * = "[zZ]one - order by zone, then by locality within zone
  2168. */
  2169. static int __parse_numa_zonelist_order(char *s)
  2170. {
  2171. if (*s == 'd' || *s == 'D') {
  2172. user_zonelist_order = ZONELIST_ORDER_DEFAULT;
  2173. } else if (*s == 'n' || *s == 'N') {
  2174. user_zonelist_order = ZONELIST_ORDER_NODE;
  2175. } else if (*s == 'z' || *s == 'Z') {
  2176. user_zonelist_order = ZONELIST_ORDER_ZONE;
  2177. } else {
  2178. printk(KERN_WARNING
  2179. "Ignoring invalid numa_zonelist_order value: "
  2180. "%s\n", s);
  2181. return -EINVAL;
  2182. }
  2183. return 0;
  2184. }
  2185. static __init int setup_numa_zonelist_order(char *s)
  2186. {
  2187. if (s)
  2188. return __parse_numa_zonelist_order(s);
  2189. return 0;
  2190. }
  2191. early_param("numa_zonelist_order", setup_numa_zonelist_order);
  2192. /*
  2193. * sysctl handler for numa_zonelist_order
  2194. */
  2195. int numa_zonelist_order_handler(ctl_table *table, int write,
  2196. void __user *buffer, size_t *length,
  2197. loff_t *ppos)
  2198. {
  2199. char saved_string[NUMA_ZONELIST_ORDER_LEN];
  2200. int ret;
  2201. static DEFINE_MUTEX(zl_order_mutex);
  2202. mutex_lock(&zl_order_mutex);
  2203. if (write)
  2204. strcpy(saved_string, (char*)table->data);
  2205. ret = proc_dostring(table, write, buffer, length, ppos);
  2206. if (ret)
  2207. goto out;
  2208. if (write) {
  2209. int oldval = user_zonelist_order;
  2210. if (__parse_numa_zonelist_order((char*)table->data)) {
  2211. /*
  2212. * bogus value. restore saved string
  2213. */
  2214. strncpy((char*)table->data, saved_string,
  2215. NUMA_ZONELIST_ORDER_LEN);
  2216. user_zonelist_order = oldval;
  2217. } else if (oldval != user_zonelist_order) {
  2218. mutex_lock(&zonelists_mutex);
  2219. build_all_zonelists(NULL);
  2220. mutex_unlock(&zonelists_mutex);
  2221. }
  2222. }
  2223. out:
  2224. mutex_unlock(&zl_order_mutex);
  2225. return ret;
  2226. }
  2227. #define MAX_NODE_LOAD (nr_online_nodes)
  2228. static int node_load[MAX_NUMNODES];
  2229. /**
  2230. * find_next_best_node - find the next node that should appear in a given node's fallback list
  2231. * @node: node whose fallback list we're appending
  2232. * @used_node_mask: nodemask_t of already used nodes
  2233. *
  2234. * We use a number of factors to determine which is the next node that should
  2235. * appear on a given node's fallback list. The node should not have appeared
  2236. * already in @node's fallback list, and it should be the next closest node
  2237. * according to the distance array (which contains arbitrary distance values
  2238. * from each node to each node in the system), and should also prefer nodes
  2239. * with no CPUs, since presumably they'll have very little allocation pressure
  2240. * on them otherwise.
  2241. * It returns -1 if no node is found.
  2242. */
  2243. static int find_next_best_node(int node, nodemask_t *used_node_mask)
  2244. {
  2245. int n, val;
  2246. int min_val = INT_MAX;
  2247. int best_node = -1;
  2248. const struct cpumask *tmp = cpumask_of_node(0);
  2249. /* Use the local node if we haven't already */
  2250. if (!node_isset(node, *used_node_mask)) {
  2251. node_set(node, *used_node_mask);
  2252. return node;
  2253. }
  2254. for_each_node_state(n, N_HIGH_MEMORY) {
  2255. /* Don't want a node to appear more than once */
  2256. if (node_isset(n, *used_node_mask))
  2257. continue;
  2258. /* Use the distance array to find the distance */
  2259. val = node_distance(node, n);
  2260. /* Penalize nodes under us ("prefer the next node") */
  2261. val += (n < node);
  2262. /* Give preference to headless and unused nodes */
  2263. tmp = cpumask_of_node(n);
  2264. if (!cpumask_empty(tmp))
  2265. val += PENALTY_FOR_NODE_WITH_CPUS;
  2266. /* Slight preference for less loaded node */
  2267. val *= (MAX_NODE_LOAD*MAX_NUMNODES);
  2268. val += node_load[n];
  2269. if (val < min_val) {
  2270. min_val = val;
  2271. best_node = n;
  2272. }
  2273. }
  2274. if (best_node >= 0)
  2275. node_set(best_node, *used_node_mask);
  2276. return best_node;
  2277. }
  2278. /*
  2279. * Build zonelists ordered by node and zones within node.
  2280. * This results in maximum locality--normal zone overflows into local
  2281. * DMA zone, if any--but risks exhausting DMA zone.
  2282. */
  2283. static void build_zonelists_in_node_order(pg_data_t *pgdat, int node)
  2284. {
  2285. int j;
  2286. struct zonelist *zonelist;
  2287. zonelist = &pgdat->node_zonelists[0];
  2288. for (j = 0; zonelist->_zonerefs[j].zone != NULL; j++)
  2289. ;
  2290. j = build_zonelists_node(NODE_DATA(node), zonelist, j,
  2291. MAX_NR_ZONES - 1);
  2292. zonelist->_zonerefs[j].zone = NULL;
  2293. zonelist->_zonerefs[j].zone_idx = 0;
  2294. }
  2295. /*
  2296. * Build gfp_thisnode zonelists
  2297. */
  2298. static void build_thisnode_zonelists(pg_data_t *pgdat)
  2299. {
  2300. int j;
  2301. struct zonelist *zonelist;
  2302. zonelist = &pgdat->node_zonelists[1];
  2303. j = build_zonelists_node(pgdat, zonelist, 0, MAX_NR_ZONES - 1);
  2304. zonelist->_zonerefs[j].zone = NULL;
  2305. zonelist->_zonerefs[j].zone_idx = 0;
  2306. }
  2307. /*
  2308. * Build zonelists ordered by zone and nodes within zones.
  2309. * This results in conserving DMA zone[s] until all Normal memory is
  2310. * exhausted, but results in overflowing to remote node while memory
  2311. * may still exist in local DMA zone.
  2312. */
  2313. static int node_order[MAX_NUMNODES];
  2314. static void build_zonelists_in_zone_order(pg_data_t *pgdat, int nr_nodes)
  2315. {
  2316. int pos, j, node;
  2317. int zone_type; /* needs to be signed */
  2318. struct zone *z;
  2319. struct zonelist *zonelist;
  2320. zonelist = &pgdat->node_zonelists[0];
  2321. pos = 0;
  2322. for (zone_type = MAX_NR_ZONES - 1; zone_type >= 0; zone_type--) {
  2323. for (j = 0; j < nr_nodes; j++) {
  2324. node = node_order[j];
  2325. z = &NODE_DATA(node)->node_zones[zone_type];
  2326. if (populated_zone(z)) {
  2327. zoneref_set_zone(z,
  2328. &zonelist->_zonerefs[pos++]);
  2329. check_highest_zone(zone_type);
  2330. }
  2331. }
  2332. }
  2333. zonelist->_zonerefs[pos].zone = NULL;
  2334. zonelist->_zonerefs[pos].zone_idx = 0;
  2335. }
  2336. static int default_zonelist_order(void)
  2337. {
  2338. int nid, zone_type;
  2339. unsigned long low_kmem_size,total_size;
  2340. struct zone *z;
  2341. int average_size;
  2342. /*
  2343. * ZONE_DMA and ZONE_DMA32 can be very small area in the system.
  2344. * If they are really small and used heavily, the system can fall
  2345. * into OOM very easily.
  2346. * This function detect ZONE_DMA/DMA32 size and configures zone order.
  2347. */
  2348. /* Is there ZONE_NORMAL ? (ex. ppc has only DMA zone..) */
  2349. low_kmem_size = 0;
  2350. total_size = 0;
  2351. for_each_online_node(nid) {
  2352. for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
  2353. z = &NODE_DATA(nid)->node_zones[zone_type];
  2354. if (populated_zone(z)) {
  2355. if (zone_type < ZONE_NORMAL)
  2356. low_kmem_size += z->present_pages;
  2357. total_size += z->present_pages;
  2358. } else if (zone_type == ZONE_NORMAL) {
  2359. /*
  2360. * If any node has only lowmem, then node order
  2361. * is preferred to allow kernel allocations
  2362. * locally; otherwise, they can easily infringe
  2363. * on other nodes when there is an abundance of
  2364. * lowmem available to allocate from.
  2365. */
  2366. return ZONELIST_ORDER_NODE;
  2367. }
  2368. }
  2369. }
  2370. if (!low_kmem_size || /* there are no DMA area. */
  2371. low_kmem_size > total_size/2) /* DMA/DMA32 is big. */
  2372. return ZONELIST_ORDER_NODE;
  2373. /*
  2374. * look into each node's config.
  2375. * If there is a node whose DMA/DMA32 memory is very big area on
  2376. * local memory, NODE_ORDER may be suitable.
  2377. */
  2378. average_size = total_size /
  2379. (nodes_weight(node_states[N_HIGH_MEMORY]) + 1);
  2380. for_each_online_node(nid) {
  2381. low_kmem_size = 0;
  2382. total_size = 0;
  2383. for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
  2384. z = &NODE_DATA(nid)->node_zones[zone_type];
  2385. if (populated_zone(z)) {
  2386. if (zone_type < ZONE_NORMAL)
  2387. low_kmem_size += z->present_pages;
  2388. total_size += z->present_pages;
  2389. }
  2390. }
  2391. if (low_kmem_size &&
  2392. total_size > average_size && /* ignore small node */
  2393. low_kmem_size > total_size * 70/100)
  2394. return ZONELIST_ORDER_NODE;
  2395. }
  2396. return ZONELIST_ORDER_ZONE;
  2397. }
  2398. static void set_zonelist_order(void)
  2399. {
  2400. if (user_zonelist_order == ZONELIST_ORDER_DEFAULT)
  2401. current_zonelist_order = default_zonelist_order();
  2402. else
  2403. current_zonelist_order = user_zonelist_order;
  2404. }
  2405. static void build_zonelists(pg_data_t *pgdat)
  2406. {
  2407. int j, node, load;
  2408. enum zone_type i;
  2409. nodemask_t used_mask;
  2410. int local_node, prev_node;
  2411. struct zonelist *zonelist;
  2412. int order = current_zonelist_order;
  2413. /* initialize zonelists */
  2414. for (i = 0; i < MAX_ZONELISTS; i++) {
  2415. zonelist = pgdat->node_zonelists + i;
  2416. zonelist->_zonerefs[0].zone = NULL;
  2417. zonelist->_zonerefs[0].zone_idx = 0;
  2418. }
  2419. /* NUMA-aware ordering of nodes */
  2420. local_node = pgdat->node_id;
  2421. load = nr_online_nodes;
  2422. prev_node = local_node;
  2423. nodes_clear(used_mask);
  2424. memset(node_order, 0, sizeof(node_order));
  2425. j = 0;
  2426. while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
  2427. int distance = node_distance(local_node, node);
  2428. /*
  2429. * If another node is sufficiently far away then it is better
  2430. * to reclaim pages in a zone before going off node.
  2431. */
  2432. if (distance > RECLAIM_DISTANCE)
  2433. zone_reclaim_mode = 1;
  2434. /*
  2435. * We don't want to pressure a particular node.
  2436. * So adding penalty to the first node in same
  2437. * distance group to make it round-robin.
  2438. */
  2439. if (distance != node_distance(local_node, prev_node))
  2440. node_load[node] = load;
  2441. prev_node = node;
  2442. load--;
  2443. if (order == ZONELIST_ORDER_NODE)
  2444. build_zonelists_in_node_order(pgdat, node);
  2445. else
  2446. node_order[j++] = node; /* remember order */
  2447. }
  2448. if (order == ZONELIST_ORDER_ZONE) {
  2449. /* calculate node order -- i.e., DMA last! */
  2450. build_zonelists_in_zone_order(pgdat, j);
  2451. }
  2452. build_thisnode_zonelists(pgdat);
  2453. }
  2454. /* Construct the zonelist performance cache - see further mmzone.h */
  2455. static void build_zonelist_cache(pg_data_t *pgdat)
  2456. {
  2457. struct zonelist *zonelist;
  2458. struct zonelist_cache *zlc;
  2459. struct zoneref *z;
  2460. zonelist = &pgdat->node_zonelists[0];
  2461. zonelist->zlcache_ptr = zlc = &zonelist->zlcache;
  2462. bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
  2463. for (z = zonelist->_zonerefs; z->zone; z++)
  2464. zlc->z_to_n[z - zonelist->_zonerefs] = zonelist_node_idx(z);
  2465. }
  2466. #else /* CONFIG_NUMA */
  2467. static void set_zonelist_order(void)
  2468. {
  2469. current_zonelist_order = ZONELIST_ORDER_ZONE;
  2470. }
  2471. static void build_zonelists(pg_data_t *pgdat)
  2472. {
  2473. int node, local_node;
  2474. enum zone_type j;
  2475. struct zonelist *zonelist;
  2476. local_node = pgdat->node_id;
  2477. zonelist = &pgdat->node_zonelists[0];
  2478. j = build_zonelists_node(pgdat, zonelist, 0, MAX_NR_ZONES - 1);
  2479. /*
  2480. * Now we build the zonelist so that it contains the zones
  2481. * of all the other nodes.
  2482. * We don't want to pressure a particular node, so when
  2483. * building the zones for node N, we make sure that the
  2484. * zones coming right after the local ones are those from
  2485. * node N+1 (modulo N)
  2486. */
  2487. for (node = local_node + 1; node < MAX_NUMNODES; node++) {
  2488. if (!node_online(node))
  2489. continue;
  2490. j = build_zonelists_node(NODE_DATA(node), zonelist, j,
  2491. MAX_NR_ZONES - 1);
  2492. }
  2493. for (node = 0; node < local_node; node++) {
  2494. if (!node_online(node))
  2495. continue;
  2496. j = build_zonelists_node(NODE_DATA(node), zonelist, j,
  2497. MAX_NR_ZONES - 1);
  2498. }
  2499. zonelist->_zonerefs[j].zone = NULL;
  2500. zonelist->_zonerefs[j].zone_idx = 0;
  2501. }
  2502. /* non-NUMA variant of zonelist performance cache - just NULL zlcache_ptr */
  2503. static void build_zonelist_cache(pg_data_t *pgdat)
  2504. {
  2505. pgdat->node_zonelists[0].zlcache_ptr = NULL;
  2506. }
  2507. #endif /* CONFIG_NUMA */
  2508. /*
  2509. * Boot pageset table. One per cpu which is going to be used for all
  2510. * zones and all nodes. The parameters will be set in such a way
  2511. * that an item put on a list will immediately be handed over to
  2512. * the buddy list. This is safe since pageset manipulation is done
  2513. * with interrupts disabled.
  2514. *
  2515. * The boot_pagesets must be kept even after bootup is complete for
  2516. * unused processors and/or zones. They do play a role for bootstrapping
  2517. * hotplugged processors.
  2518. *
  2519. * zoneinfo_show() and maybe other functions do
  2520. * not check if the processor is online before following the pageset pointer.
  2521. * Other parts of the kernel may not check if the zone is available.
  2522. */
  2523. static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
  2524. static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
  2525. static void setup_zone_pageset(struct zone *zone);
  2526. /*
  2527. * Global mutex to protect against size modification of zonelists
  2528. * as well as to serialize pageset setup for the new populated zone.
  2529. */
  2530. DEFINE_MUTEX(zonelists_mutex);
  2531. /* return values int ....just for stop_machine() */
  2532. static __init_refok int __build_all_zonelists(void *data)
  2533. {
  2534. int nid;
  2535. int cpu;
  2536. #ifdef CONFIG_NUMA
  2537. memset(node_load, 0, sizeof(node_load));
  2538. #endif
  2539. for_each_online_node(nid) {
  2540. pg_data_t *pgdat = NODE_DATA(nid);
  2541. build_zonelists(pgdat);
  2542. build_zonelist_cache(pgdat);
  2543. }
  2544. #ifdef CONFIG_MEMORY_HOTPLUG
  2545. /* Setup real pagesets for the new zone */
  2546. if (data) {
  2547. struct zone *zone = data;
  2548. setup_zone_pageset(zone);
  2549. }
  2550. #endif
  2551. /*
  2552. * Initialize the boot_pagesets that are going to be used
  2553. * for bootstrapping processors. The real pagesets for
  2554. * each zone will be allocated later when the per cpu
  2555. * allocator is available.
  2556. *
  2557. * boot_pagesets are used also for bootstrapping offline
  2558. * cpus if the system is already booted because the pagesets
  2559. * are needed to initialize allocators on a specific cpu too.
  2560. * F.e. the percpu allocator needs the page allocator which
  2561. * needs the percpu allocator in order to allocate its pagesets
  2562. * (a chicken-egg dilemma).
  2563. */
  2564. for_each_possible_cpu(cpu)
  2565. setup_pageset(&per_cpu(boot_pageset, cpu), 0);
  2566. return 0;
  2567. }
  2568. /*
  2569. * Called with zonelists_mutex held always
  2570. * unless system_state == SYSTEM_BOOTING.
  2571. */
  2572. void build_all_zonelists(void *data)
  2573. {
  2574. set_zonelist_order();
  2575. if (system_state == SYSTEM_BOOTING) {
  2576. __build_all_zonelists(NULL);
  2577. mminit_verify_zonelist();
  2578. cpuset_init_current_mems_allowed();
  2579. } else {
  2580. /* we have to stop all cpus to guarantee there is no user
  2581. of zonelist */
  2582. stop_machine(__build_all_zonelists, data, NULL);
  2583. /* cpuset refresh routine should be here */
  2584. }
  2585. vm_total_pages = nr_free_pagecache_pages();
  2586. /*
  2587. * Disable grouping by mobility if the number of pages in the
  2588. * system is too low to allow the mechanism to work. It would be
  2589. * more accurate, but expensive to check per-zone. This check is
  2590. * made on memory-hotadd so a system can start with mobility
  2591. * disabled and enable it later
  2592. */
  2593. if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
  2594. page_group_by_mobility_disabled = 1;
  2595. else
  2596. page_group_by_mobility_disabled = 0;
  2597. printk("Built %i zonelists in %s order, mobility grouping %s. "
  2598. "Total pages: %ld\n",
  2599. nr_online_nodes,
  2600. zonelist_order_name[current_zonelist_order],
  2601. page_group_by_mobility_disabled ? "off" : "on",
  2602. vm_total_pages);
  2603. #ifdef CONFIG_NUMA
  2604. printk("Policy zone: %s\n", zone_names[policy_zone]);
  2605. #endif
  2606. }
  2607. /*
  2608. * Helper functions to size the waitqueue hash table.
  2609. * Essentially these want to choose hash table sizes sufficiently
  2610. * large so that collisions trying to wait on pages are rare.
  2611. * But in fact, the number of active page waitqueues on typical
  2612. * systems is ridiculously low, less than 200. So this is even
  2613. * conservative, even though it seems large.
  2614. *
  2615. * The constant PAGES_PER_WAITQUEUE specifies the ratio of pages to
  2616. * waitqueues, i.e. the size of the waitq table given the number of pages.
  2617. */
  2618. #define PAGES_PER_WAITQUEUE 256
  2619. #ifndef CONFIG_MEMORY_HOTPLUG
  2620. static inline unsigned long wait_table_hash_nr_entries(unsigned long pages)
  2621. {
  2622. unsigned long size = 1;
  2623. pages /= PAGES_PER_WAITQUEUE;
  2624. while (size < pages)
  2625. size <<= 1;
  2626. /*
  2627. * Once we have dozens or even hundreds of threads sleeping
  2628. * on IO we've got bigger problems than wait queue collision.
  2629. * Limit the size of the wait table to a reasonable size.
  2630. */
  2631. size = min(size, 4096UL);
  2632. return max(size, 4UL);
  2633. }
  2634. #else
  2635. /*
  2636. * A zone's size might be changed by hot-add, so it is not possible to determine
  2637. * a suitable size for its wait_table. So we use the maximum size now.
  2638. *
  2639. * The max wait table size = 4096 x sizeof(wait_queue_head_t). ie:
  2640. *
  2641. * i386 (preemption config) : 4096 x 16 = 64Kbyte.
  2642. * ia64, x86-64 (no preemption): 4096 x 20 = 80Kbyte.
  2643. * ia64, x86-64 (preemption) : 4096 x 24 = 96Kbyte.
  2644. *
  2645. * The maximum entries are prepared when a zone's memory is (512K + 256) pages
  2646. * or more by the traditional way. (See above). It equals:
  2647. *
  2648. * i386, x86-64, powerpc(4K page size) : = ( 2G + 1M)byte.
  2649. * ia64(16K page size) : = ( 8G + 4M)byte.
  2650. * powerpc (64K page size) : = (32G +16M)byte.
  2651. */
  2652. static inline unsigned long wait_table_hash_nr_entries(unsigned long pages)
  2653. {
  2654. return 4096UL;
  2655. }
  2656. #endif
  2657. /*
  2658. * This is an integer logarithm so that shifts can be used later
  2659. * to extract the more random high bits from the multiplicative
  2660. * hash function before the remainder is taken.
  2661. */
  2662. static inline unsigned long wait_table_bits(unsigned long size)
  2663. {
  2664. return ffz(~size);
  2665. }
  2666. #define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1))
  2667. /*
  2668. * Mark a number of pageblocks as MIGRATE_RESERVE. The number
  2669. * of blocks reserved is based on min_wmark_pages(zone). The memory within
  2670. * the reserve will tend to store contiguous free pages. Setting min_free_kbytes
  2671. * higher will lead to a bigger reserve which will get freed as contiguous
  2672. * blocks as reclaim kicks in
  2673. */
  2674. static void setup_zone_migrate_reserve(struct zone *zone)
  2675. {
  2676. unsigned long start_pfn, pfn, end_pfn;
  2677. struct page *page;
  2678. unsigned long block_migratetype;
  2679. int reserve;
  2680. /* Get the start pfn, end pfn and the number of blocks to reserve */
  2681. start_pfn = zone->zone_start_pfn;
  2682. end_pfn = start_pfn + zone->spanned_pages;
  2683. reserve = roundup(min_wmark_pages(zone), pageblock_nr_pages) >>
  2684. pageblock_order;
  2685. /*
  2686. * Reserve blocks are generally in place to help high-order atomic
  2687. * allocations that are short-lived. A min_free_kbytes value that
  2688. * would result in more than 2 reserve blocks for atomic allocations
  2689. * is assumed to be in place to help anti-fragmentation for the
  2690. * future allocation of hugepages at runtime.
  2691. */
  2692. reserve = min(2, reserve);
  2693. for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
  2694. if (!pfn_valid(pfn))
  2695. continue;
  2696. page = pfn_to_page(pfn);
  2697. /* Watch out for overlapping nodes */
  2698. if (page_to_nid(page) != zone_to_nid(zone))
  2699. continue;
  2700. /* Blocks with reserved pages will never free, skip them. */
  2701. if (PageReserved(page))
  2702. continue;
  2703. block_migratetype = get_pageblock_migratetype(page);
  2704. /* If this block is reserved, account for it */
  2705. if (reserve > 0 && block_migratetype == MIGRATE_RESERVE) {
  2706. reserve--;
  2707. continue;
  2708. }
  2709. /* Suitable for reserving if this block is movable */
  2710. if (reserve > 0 && block_migratetype == MIGRATE_MOVABLE) {
  2711. set_pageblock_migratetype(page, MIGRATE_RESERVE);
  2712. move_freepages_block(zone, page, MIGRATE_RESERVE);
  2713. reserve--;
  2714. continue;
  2715. }
  2716. /*
  2717. * If the reserve is met and this is a previous reserved block,
  2718. * take it back
  2719. */
  2720. if (block_migratetype == MIGRATE_RESERVE) {
  2721. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  2722. move_freepages_block(zone, page, MIGRATE_MOVABLE);
  2723. }
  2724. }
  2725. }
  2726. /*
  2727. * Initially all pages are reserved - free ones are freed
  2728. * up by free_all_bootmem() once the early boot process is
  2729. * done. Non-atomic initialization, single-pass.
  2730. */
  2731. void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
  2732. unsigned long start_pfn, enum memmap_context context)
  2733. {
  2734. struct page *page;
  2735. unsigned long end_pfn = start_pfn + size;
  2736. unsigned long pfn;
  2737. struct zone *z;
  2738. if (highest_memmap_pfn < end_pfn - 1)
  2739. highest_memmap_pfn = end_pfn - 1;
  2740. z = &NODE_DATA(nid)->node_zones[zone];
  2741. for (pfn = start_pfn; pfn < end_pfn; pfn++) {
  2742. /*
  2743. * There can be holes in boot-time mem_map[]s
  2744. * handed to this function. They do not
  2745. * exist on hotplugged memory.
  2746. */
  2747. if (context == MEMMAP_EARLY) {
  2748. if (!early_pfn_valid(pfn))
  2749. continue;
  2750. if (!early_pfn_in_nid(pfn, nid))
  2751. continue;
  2752. }
  2753. page = pfn_to_page(pfn);
  2754. set_page_links(page, zone, nid, pfn);
  2755. mminit_verify_page_links(page, zone, nid, pfn);
  2756. init_page_count(page);
  2757. reset_page_mapcount(page);
  2758. SetPageReserved(page);
  2759. /*
  2760. * Mark the block movable so that blocks are reserved for
  2761. * movable at startup. This will force kernel allocations
  2762. * to reserve their blocks rather than leaking throughout
  2763. * the address space during boot when many long-lived
  2764. * kernel allocations are made. Later some blocks near
  2765. * the start are marked MIGRATE_RESERVE by
  2766. * setup_zone_migrate_reserve()
  2767. *
  2768. * bitmap is created for zone's valid pfn range. but memmap
  2769. * can be created for invalid pages (for alignment)
  2770. * check here not to call set_pageblock_migratetype() against
  2771. * pfn out of zone.
  2772. */
  2773. if ((z->zone_start_pfn <= pfn)
  2774. && (pfn < z->zone_start_pfn + z->spanned_pages)
  2775. && !(pfn & (pageblock_nr_pages - 1)))
  2776. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  2777. INIT_LIST_HEAD(&page->lru);
  2778. #ifdef WANT_PAGE_VIRTUAL
  2779. /* The shift won't overflow because ZONE_NORMAL is below 4G. */
  2780. if (!is_highmem_idx(zone))
  2781. set_page_address(page, __va(pfn << PAGE_SHIFT));
  2782. #endif
  2783. }
  2784. }
  2785. static void __meminit zone_init_free_lists(struct zone *zone)
  2786. {
  2787. int order, t;
  2788. for_each_migratetype_order(order, t) {
  2789. INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
  2790. zone->free_area[order].nr_free = 0;
  2791. }
  2792. }
  2793. #ifndef __HAVE_ARCH_MEMMAP_INIT
  2794. #define memmap_init(size, nid, zone, start_pfn) \
  2795. memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY)
  2796. #endif
  2797. static int zone_batchsize(struct zone *zone)
  2798. {
  2799. #ifdef CONFIG_MMU
  2800. int batch;
  2801. /*
  2802. * The per-cpu-pages pools are set to around 1000th of the
  2803. * size of the zone. But no more than 1/2 of a meg.
  2804. *
  2805. * OK, so we don't know how big the cache is. So guess.
  2806. */
  2807. batch = zone->present_pages / 1024;
  2808. if (batch * PAGE_SIZE > 512 * 1024)
  2809. batch = (512 * 1024) / PAGE_SIZE;
  2810. batch /= 4; /* We effectively *= 4 below */
  2811. if (batch < 1)
  2812. batch = 1;
  2813. /*
  2814. * Clamp the batch to a 2^n - 1 value. Having a power
  2815. * of 2 value was found to be more likely to have
  2816. * suboptimal cache aliasing properties in some cases.
  2817. *
  2818. * For example if 2 tasks are alternately allocating
  2819. * batches of pages, one task can end up with a lot
  2820. * of pages of one half of the possible page colors
  2821. * and the other with pages of the other colors.
  2822. */
  2823. batch = rounddown_pow_of_two(batch + batch/2) - 1;
  2824. return batch;
  2825. #else
  2826. /* The deferral and batching of frees should be suppressed under NOMMU
  2827. * conditions.
  2828. *
  2829. * The problem is that NOMMU needs to be able to allocate large chunks
  2830. * of contiguous memory as there's no hardware page translation to
  2831. * assemble apparent contiguous memory from discontiguous pages.
  2832. *
  2833. * Queueing large contiguous runs of pages for batching, however,
  2834. * causes the pages to actually be freed in smaller chunks. As there
  2835. * can be a significant delay between the individual batches being
  2836. * recycled, this leads to the once large chunks of space being
  2837. * fragmented and becoming unavailable for high-order allocations.
  2838. */
  2839. return 0;
  2840. #endif
  2841. }
  2842. static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
  2843. {
  2844. struct per_cpu_pages *pcp;
  2845. int migratetype;
  2846. memset(p, 0, sizeof(*p));
  2847. pcp = &p->pcp;
  2848. pcp->count = 0;
  2849. pcp->high = 6 * batch;
  2850. pcp->batch = max(1UL, 1 * batch);
  2851. for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++)
  2852. INIT_LIST_HEAD(&pcp->lists[migratetype]);
  2853. }
  2854. /*
  2855. * setup_pagelist_highmark() sets the high water mark for hot per_cpu_pagelist
  2856. * to the value high for the pageset p.
  2857. */
  2858. static void setup_pagelist_highmark(struct per_cpu_pageset *p,
  2859. unsigned long high)
  2860. {
  2861. struct per_cpu_pages *pcp;
  2862. pcp = &p->pcp;
  2863. pcp->high = high;
  2864. pcp->batch = max(1UL, high/4);
  2865. if ((high/4) > (PAGE_SHIFT * 8))
  2866. pcp->batch = PAGE_SHIFT * 8;
  2867. }
  2868. static __meminit void setup_zone_pageset(struct zone *zone)
  2869. {
  2870. int cpu;
  2871. zone->pageset = alloc_percpu(struct per_cpu_pageset);
  2872. for_each_possible_cpu(cpu) {
  2873. struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);
  2874. setup_pageset(pcp, zone_batchsize(zone));
  2875. if (percpu_pagelist_fraction)
  2876. setup_pagelist_highmark(pcp,
  2877. (zone->present_pages /
  2878. percpu_pagelist_fraction));
  2879. }
  2880. }
  2881. /*
  2882. * Allocate per cpu pagesets and initialize them.
  2883. * Before this call only boot pagesets were available.
  2884. */
  2885. void __init setup_per_cpu_pageset(void)
  2886. {
  2887. struct zone *zone;
  2888. for_each_populated_zone(zone)
  2889. setup_zone_pageset(zone);
  2890. }
  2891. static noinline __init_refok
  2892. int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages)
  2893. {
  2894. int i;
  2895. struct pglist_data *pgdat = zone->zone_pgdat;
  2896. size_t alloc_size;
  2897. /*
  2898. * The per-page waitqueue mechanism uses hashed waitqueues
  2899. * per zone.
  2900. */
  2901. zone->wait_table_hash_nr_entries =
  2902. wait_table_hash_nr_entries(zone_size_pages);
  2903. zone->wait_table_bits =
  2904. wait_table_bits(zone->wait_table_hash_nr_entries);
  2905. alloc_size = zone->wait_table_hash_nr_entries
  2906. * sizeof(wait_queue_head_t);
  2907. if (!slab_is_available()) {
  2908. zone->wait_table = (wait_queue_head_t *)
  2909. alloc_bootmem_node(pgdat, alloc_size);
  2910. } else {
  2911. /*
  2912. * This case means that a zone whose size was 0 gets new memory
  2913. * via memory hot-add.
  2914. * But it may be the case that a new node was hot-added. In
  2915. * this case vmalloc() will not be able to use this new node's
  2916. * memory - this wait_table must be initialized to use this new
  2917. * node itself as well.
  2918. * To use this new node's memory, further consideration will be
  2919. * necessary.
  2920. */
  2921. zone->wait_table = vmalloc(alloc_size);
  2922. }
  2923. if (!zone->wait_table)
  2924. return -ENOMEM;
  2925. for(i = 0; i < zone->wait_table_hash_nr_entries; ++i)
  2926. init_waitqueue_head(zone->wait_table + i);
  2927. return 0;
  2928. }
  2929. static int __zone_pcp_update(void *data)
  2930. {
  2931. struct zone *zone = data;
  2932. int cpu;
  2933. unsigned long batch = zone_batchsize(zone), flags;
  2934. for_each_possible_cpu(cpu) {
  2935. struct per_cpu_pageset *pset;
  2936. struct per_cpu_pages *pcp;
  2937. pset = per_cpu_ptr(zone->pageset, cpu);
  2938. pcp = &pset->pcp;
  2939. local_irq_save(flags);
  2940. free_pcppages_bulk(zone, pcp->count, pcp);
  2941. setup_pageset(pset, batch);
  2942. local_irq_restore(flags);
  2943. }
  2944. return 0;
  2945. }
  2946. void zone_pcp_update(struct zone *zone)
  2947. {
  2948. stop_machine(__zone_pcp_update, zone, NULL);
  2949. }
  2950. static __meminit void zone_pcp_init(struct zone *zone)
  2951. {
  2952. /*
  2953. * per cpu subsystem is not up at this point. The following code
  2954. * relies on the ability of the linker to provide the
  2955. * offset of a (static) per cpu variable into the per cpu area.
  2956. */
  2957. zone->pageset = &boot_pageset;
  2958. if (zone->present_pages)
  2959. printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n",
  2960. zone->name, zone->present_pages,
  2961. zone_batchsize(zone));
  2962. }
  2963. __meminit int init_currently_empty_zone(struct zone *zone,
  2964. unsigned long zone_start_pfn,
  2965. unsigned long size,
  2966. enum memmap_context context)
  2967. {
  2968. struct pglist_data *pgdat = zone->zone_pgdat;
  2969. int ret;
  2970. ret = zone_wait_table_init(zone, size);
  2971. if (ret)
  2972. return ret;
  2973. pgdat->nr_zones = zone_idx(zone) + 1;
  2974. zone->zone_start_pfn = zone_start_pfn;
  2975. mminit_dprintk(MMINIT_TRACE, "memmap_init",
  2976. "Initialising map node %d zone %lu pfns %lu -> %lu\n",
  2977. pgdat->node_id,
  2978. (unsigned long)zone_idx(zone),
  2979. zone_start_pfn, (zone_start_pfn + size));
  2980. zone_init_free_lists(zone);
  2981. return 0;
  2982. }
  2983. #ifdef CONFIG_ARCH_POPULATES_NODE_MAP
  2984. /*
  2985. * Basic iterator support. Return the first range of PFNs for a node
  2986. * Note: nid == MAX_NUMNODES returns first region regardless of node
  2987. */
  2988. static int __meminit first_active_region_index_in_nid(int nid)
  2989. {
  2990. int i;
  2991. for (i = 0; i < nr_nodemap_entries; i++)
  2992. if (nid == MAX_NUMNODES || early_node_map[i].nid == nid)
  2993. return i;
  2994. return -1;
  2995. }
  2996. /*
  2997. * Basic iterator support. Return the next active range of PFNs for a node
  2998. * Note: nid == MAX_NUMNODES returns next region regardless of node
  2999. */
  3000. static int __meminit next_active_region_index_in_nid(int index, int nid)
  3001. {
  3002. for (index = index + 1; index < nr_nodemap_entries; index++)
  3003. if (nid == MAX_NUMNODES || early_node_map[index].nid == nid)
  3004. return index;
  3005. return -1;
  3006. }
  3007. #ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
  3008. /*
  3009. * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
  3010. * Architectures may implement their own version but if add_active_range()
  3011. * was used and there are no special requirements, this is a convenient
  3012. * alternative
  3013. */
  3014. int __meminit __early_pfn_to_nid(unsigned long pfn)
  3015. {
  3016. int i;
  3017. for (i = 0; i < nr_nodemap_entries; i++) {
  3018. unsigned long start_pfn = early_node_map[i].start_pfn;
  3019. unsigned long end_pfn = early_node_map[i].end_pfn;
  3020. if (start_pfn <= pfn && pfn < end_pfn)
  3021. return early_node_map[i].nid;
  3022. }
  3023. /* This is a memory hole */
  3024. return -1;
  3025. }
  3026. #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
  3027. int __meminit early_pfn_to_nid(unsigned long pfn)
  3028. {
  3029. int nid;
  3030. nid = __early_pfn_to_nid(pfn);
  3031. if (nid >= 0)
  3032. return nid;
  3033. /* just returns 0 */
  3034. return 0;
  3035. }
  3036. #ifdef CONFIG_NODES_SPAN_OTHER_NODES
  3037. bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
  3038. {
  3039. int nid;
  3040. nid = __early_pfn_to_nid(pfn);
  3041. if (nid >= 0 && nid != node)
  3042. return false;
  3043. return true;
  3044. }
  3045. #endif
  3046. /* Basic iterator support to walk early_node_map[] */
  3047. #define for_each_active_range_index_in_nid(i, nid) \
  3048. for (i = first_active_region_index_in_nid(nid); i != -1; \
  3049. i = next_active_region_index_in_nid(i, nid))
  3050. /**
  3051. * free_bootmem_with_active_regions - Call free_bootmem_node for each active range
  3052. * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed.
  3053. * @max_low_pfn: The highest PFN that will be passed to free_bootmem_node
  3054. *
  3055. * If an architecture guarantees that all ranges registered with
  3056. * add_active_ranges() contain no holes and may be freed, this
  3057. * this function may be used instead of calling free_bootmem() manually.
  3058. */
  3059. void __init free_bootmem_with_active_regions(int nid,
  3060. unsigned long max_low_pfn)
  3061. {
  3062. int i;
  3063. for_each_active_range_index_in_nid(i, nid) {
  3064. unsigned long size_pages = 0;
  3065. unsigned long end_pfn = early_node_map[i].end_pfn;
  3066. if (early_node_map[i].start_pfn >= max_low_pfn)
  3067. continue;
  3068. if (end_pfn > max_low_pfn)
  3069. end_pfn = max_low_pfn;
  3070. size_pages = end_pfn - early_node_map[i].start_pfn;
  3071. free_bootmem_node(NODE_DATA(early_node_map[i].nid),
  3072. PFN_PHYS(early_node_map[i].start_pfn),
  3073. size_pages << PAGE_SHIFT);
  3074. }
  3075. }
  3076. int __init add_from_early_node_map(struct range *range, int az,
  3077. int nr_range, int nid)
  3078. {
  3079. int i;
  3080. u64 start, end;
  3081. /* need to go over early_node_map to find out good range for node */
  3082. for_each_active_range_index_in_nid(i, nid) {
  3083. start = early_node_map[i].start_pfn;
  3084. end = early_node_map[i].end_pfn;
  3085. nr_range = add_range(range, az, nr_range, start, end);
  3086. }
  3087. return nr_range;
  3088. }
  3089. #ifdef CONFIG_NO_BOOTMEM
  3090. void * __init __alloc_memory_core_early(int nid, u64 size, u64 align,
  3091. u64 goal, u64 limit)
  3092. {
  3093. int i;
  3094. void *ptr;
  3095. /* need to go over early_node_map to find out good range for node */
  3096. for_each_active_range_index_in_nid(i, nid) {
  3097. u64 addr;
  3098. u64 ei_start, ei_last;
  3099. ei_last = early_node_map[i].end_pfn;
  3100. ei_last <<= PAGE_SHIFT;
  3101. ei_start = early_node_map[i].start_pfn;
  3102. ei_start <<= PAGE_SHIFT;
  3103. addr = find_early_area(ei_start, ei_last,
  3104. goal, limit, size, align);
  3105. if (addr == -1ULL)
  3106. continue;
  3107. #if 0
  3108. printk(KERN_DEBUG "alloc (nid=%d %llx - %llx) (%llx - %llx) %llx %llx => %llx\n",
  3109. nid,
  3110. ei_start, ei_last, goal, limit, size,
  3111. align, addr);
  3112. #endif
  3113. ptr = phys_to_virt(addr);
  3114. memset(ptr, 0, size);
  3115. reserve_early_without_check(addr, addr + size, "BOOTMEM");
  3116. return ptr;
  3117. }
  3118. return NULL;
  3119. }
  3120. #endif
  3121. void __init work_with_active_regions(int nid, work_fn_t work_fn, void *data)
  3122. {
  3123. int i;
  3124. int ret;
  3125. for_each_active_range_index_in_nid(i, nid) {
  3126. ret = work_fn(early_node_map[i].start_pfn,
  3127. early_node_map[i].end_pfn, data);
  3128. if (ret)
  3129. break;
  3130. }
  3131. }
  3132. /**
  3133. * sparse_memory_present_with_active_regions - Call memory_present for each active range
  3134. * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used.
  3135. *
  3136. * If an architecture guarantees that all ranges registered with
  3137. * add_active_ranges() contain no holes and may be freed, this
  3138. * function may be used instead of calling memory_present() manually.
  3139. */
  3140. void __init sparse_memory_present_with_active_regions(int nid)
  3141. {
  3142. int i;
  3143. for_each_active_range_index_in_nid(i, nid)
  3144. memory_present(early_node_map[i].nid,
  3145. early_node_map[i].start_pfn,
  3146. early_node_map[i].end_pfn);
  3147. }
  3148. /**
  3149. * get_pfn_range_for_nid - Return the start and end page frames for a node
  3150. * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
  3151. * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
  3152. * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
  3153. *
  3154. * It returns the start and end page frame of a node based on information
  3155. * provided by an arch calling add_active_range(). If called for a node
  3156. * with no available memory, a warning is printed and the start and end
  3157. * PFNs will be 0.
  3158. */
  3159. void __meminit get_pfn_range_for_nid(unsigned int nid,
  3160. unsigned long *start_pfn, unsigned long *end_pfn)
  3161. {
  3162. int i;
  3163. *start_pfn = -1UL;
  3164. *end_pfn = 0;
  3165. for_each_active_range_index_in_nid(i, nid) {
  3166. *start_pfn = min(*start_pfn, early_node_map[i].start_pfn);
  3167. *end_pfn = max(*end_pfn, early_node_map[i].end_pfn);
  3168. }
  3169. if (*start_pfn == -1UL)
  3170. *start_pfn = 0;
  3171. }
  3172. /*
  3173. * This finds a zone that can be used for ZONE_MOVABLE pages. The
  3174. * assumption is made that zones within a node are ordered in monotonic
  3175. * increasing memory addresses so that the "highest" populated zone is used
  3176. */
  3177. static void __init find_usable_zone_for_movable(void)
  3178. {
  3179. int zone_index;
  3180. for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
  3181. if (zone_index == ZONE_MOVABLE)
  3182. continue;
  3183. if (arch_zone_highest_possible_pfn[zone_index] >
  3184. arch_zone_lowest_possible_pfn[zone_index])
  3185. break;
  3186. }
  3187. VM_BUG_ON(zone_index == -1);
  3188. movable_zone = zone_index;
  3189. }
  3190. /*
  3191. * The zone ranges provided by the architecture do not include ZONE_MOVABLE
  3192. * because it is sized independant of architecture. Unlike the other zones,
  3193. * the starting point for ZONE_MOVABLE is not fixed. It may be different
  3194. * in each node depending on the size of each node and how evenly kernelcore
  3195. * is distributed. This helper function adjusts the zone ranges
  3196. * provided by the architecture for a given node by using the end of the
  3197. * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
  3198. * zones within a node are in order of monotonic increases memory addresses
  3199. */
  3200. static void __meminit adjust_zone_range_for_zone_movable(int nid,
  3201. unsigned long zone_type,
  3202. unsigned long node_start_pfn,
  3203. unsigned long node_end_pfn,
  3204. unsigned long *zone_start_pfn,
  3205. unsigned long *zone_end_pfn)
  3206. {
  3207. /* Only adjust if ZONE_MOVABLE is on this node */
  3208. if (zone_movable_pfn[nid]) {
  3209. /* Size ZONE_MOVABLE */
  3210. if (zone_type == ZONE_MOVABLE) {
  3211. *zone_start_pfn = zone_movable_pfn[nid];
  3212. *zone_end_pfn = min(node_end_pfn,
  3213. arch_zone_highest_possible_pfn[movable_zone]);
  3214. /* Adjust for ZONE_MOVABLE starting within this range */
  3215. } else if (*zone_start_pfn < zone_movable_pfn[nid] &&
  3216. *zone_end_pfn > zone_movable_pfn[nid]) {
  3217. *zone_end_pfn = zone_movable_pfn[nid];
  3218. /* Check if this whole range is within ZONE_MOVABLE */
  3219. } else if (*zone_start_pfn >= zone_movable_pfn[nid])
  3220. *zone_start_pfn = *zone_end_pfn;
  3221. }
  3222. }
  3223. /*
  3224. * Return the number of pages a zone spans in a node, including holes
  3225. * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
  3226. */
  3227. static unsigned long __meminit zone_spanned_pages_in_node(int nid,
  3228. unsigned long zone_type,
  3229. unsigned long *ignored)
  3230. {
  3231. unsigned long node_start_pfn, node_end_pfn;
  3232. unsigned long zone_start_pfn, zone_end_pfn;
  3233. /* Get the start and end of the node and zone */
  3234. get_pfn_range_for_nid(nid, &node_start_pfn, &node_end_pfn);
  3235. zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type];
  3236. zone_end_pfn = arch_zone_highest_possible_pfn[zone_type];
  3237. adjust_zone_range_for_zone_movable(nid, zone_type,
  3238. node_start_pfn, node_end_pfn,
  3239. &zone_start_pfn, &zone_end_pfn);
  3240. /* Check that this node has pages within the zone's required range */
  3241. if (zone_end_pfn < node_start_pfn || zone_start_pfn > node_end_pfn)
  3242. return 0;
  3243. /* Move the zone boundaries inside the node if necessary */
  3244. zone_end_pfn = min(zone_end_pfn, node_end_pfn);
  3245. zone_start_pfn = max(zone_start_pfn, node_start_pfn);
  3246. /* Return the spanned pages */
  3247. return zone_end_pfn - zone_start_pfn;
  3248. }
  3249. /*
  3250. * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
  3251. * then all holes in the requested range will be accounted for.
  3252. */
  3253. unsigned long __meminit __absent_pages_in_range(int nid,
  3254. unsigned long range_start_pfn,
  3255. unsigned long range_end_pfn)
  3256. {
  3257. int i = 0;
  3258. unsigned long prev_end_pfn = 0, hole_pages = 0;
  3259. unsigned long start_pfn;
  3260. /* Find the end_pfn of the first active range of pfns in the node */
  3261. i = first_active_region_index_in_nid(nid);
  3262. if (i == -1)
  3263. return 0;
  3264. prev_end_pfn = min(early_node_map[i].start_pfn, range_end_pfn);
  3265. /* Account for ranges before physical memory on this node */
  3266. if (early_node_map[i].start_pfn > range_start_pfn)
  3267. hole_pages = prev_end_pfn - range_start_pfn;
  3268. /* Find all holes for the zone within the node */
  3269. for (; i != -1; i = next_active_region_index_in_nid(i, nid)) {
  3270. /* No need to continue if prev_end_pfn is outside the zone */
  3271. if (prev_end_pfn >= range_end_pfn)
  3272. break;
  3273. /* Make sure the end of the zone is not within the hole */
  3274. start_pfn = min(early_node_map[i].start_pfn, range_end_pfn);
  3275. prev_end_pfn = max(prev_end_pfn, range_start_pfn);
  3276. /* Update the hole size cound and move on */
  3277. if (start_pfn > range_start_pfn) {
  3278. BUG_ON(prev_end_pfn > start_pfn);
  3279. hole_pages += start_pfn - prev_end_pfn;
  3280. }
  3281. prev_end_pfn = early_node_map[i].end_pfn;
  3282. }
  3283. /* Account for ranges past physical memory on this node */
  3284. if (range_end_pfn > prev_end_pfn)
  3285. hole_pages += range_end_pfn -
  3286. max(range_start_pfn, prev_end_pfn);
  3287. return hole_pages;
  3288. }
  3289. /**
  3290. * absent_pages_in_range - Return number of page frames in holes within a range
  3291. * @start_pfn: The start PFN to start searching for holes
  3292. * @end_pfn: The end PFN to stop searching for holes
  3293. *
  3294. * It returns the number of pages frames in memory holes within a range.
  3295. */
  3296. unsigned long __init absent_pages_in_range(unsigned long start_pfn,
  3297. unsigned long end_pfn)
  3298. {
  3299. return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
  3300. }
  3301. /* Return the number of page frames in holes in a zone on a node */
  3302. static unsigned long __meminit zone_absent_pages_in_node(int nid,
  3303. unsigned long zone_type,
  3304. unsigned long *ignored)
  3305. {
  3306. unsigned long node_start_pfn, node_end_pfn;
  3307. unsigned long zone_start_pfn, zone_end_pfn;
  3308. get_pfn_range_for_nid(nid, &node_start_pfn, &node_end_pfn);
  3309. zone_start_pfn = max(arch_zone_lowest_possible_pfn[zone_type],
  3310. node_start_pfn);
  3311. zone_end_pfn = min(arch_zone_highest_possible_pfn[zone_type],
  3312. node_end_pfn);
  3313. adjust_zone_range_for_zone_movable(nid, zone_type,
  3314. node_start_pfn, node_end_pfn,
  3315. &zone_start_pfn, &zone_end_pfn);
  3316. return __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
  3317. }
  3318. #else
  3319. static inline unsigned long __meminit zone_spanned_pages_in_node(int nid,
  3320. unsigned long zone_type,
  3321. unsigned long *zones_size)
  3322. {
  3323. return zones_size[zone_type];
  3324. }
  3325. static inline unsigned long __meminit zone_absent_pages_in_node(int nid,
  3326. unsigned long zone_type,
  3327. unsigned long *zholes_size)
  3328. {
  3329. if (!zholes_size)
  3330. return 0;
  3331. return zholes_size[zone_type];
  3332. }
  3333. #endif
  3334. static void __meminit calculate_node_totalpages(struct pglist_data *pgdat,
  3335. unsigned long *zones_size, unsigned long *zholes_size)
  3336. {
  3337. unsigned long realtotalpages, totalpages = 0;
  3338. enum zone_type i;
  3339. for (i = 0; i < MAX_NR_ZONES; i++)
  3340. totalpages += zone_spanned_pages_in_node(pgdat->node_id, i,
  3341. zones_size);
  3342. pgdat->node_spanned_pages = totalpages;
  3343. realtotalpages = totalpages;
  3344. for (i = 0; i < MAX_NR_ZONES; i++)
  3345. realtotalpages -=
  3346. zone_absent_pages_in_node(pgdat->node_id, i,
  3347. zholes_size);
  3348. pgdat->node_present_pages = realtotalpages;
  3349. printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id,
  3350. realtotalpages);
  3351. }
  3352. #ifndef CONFIG_SPARSEMEM
  3353. /*
  3354. * Calculate the size of the zone->blockflags rounded to an unsigned long
  3355. * Start by making sure zonesize is a multiple of pageblock_order by rounding
  3356. * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
  3357. * round what is now in bits to nearest long in bits, then return it in
  3358. * bytes.
  3359. */
  3360. static unsigned long __init usemap_size(unsigned long zonesize)
  3361. {
  3362. unsigned long usemapsize;
  3363. usemapsize = roundup(zonesize, pageblock_nr_pages);
  3364. usemapsize = usemapsize >> pageblock_order;
  3365. usemapsize *= NR_PAGEBLOCK_BITS;
  3366. usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
  3367. return usemapsize / 8;
  3368. }
  3369. static void __init setup_usemap(struct pglist_data *pgdat,
  3370. struct zone *zone, unsigned long zonesize)
  3371. {
  3372. unsigned long usemapsize = usemap_size(zonesize);
  3373. zone->pageblock_flags = NULL;
  3374. if (usemapsize)
  3375. zone->pageblock_flags = alloc_bootmem_node(pgdat, usemapsize);
  3376. }
  3377. #else
  3378. static void inline setup_usemap(struct pglist_data *pgdat,
  3379. struct zone *zone, unsigned long zonesize) {}
  3380. #endif /* CONFIG_SPARSEMEM */
  3381. #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
  3382. /* Return a sensible default order for the pageblock size. */
  3383. static inline int pageblock_default_order(void)
  3384. {
  3385. if (HPAGE_SHIFT > PAGE_SHIFT)
  3386. return HUGETLB_PAGE_ORDER;
  3387. return MAX_ORDER-1;
  3388. }
  3389. /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
  3390. static inline void __init set_pageblock_order(unsigned int order)
  3391. {
  3392. /* Check that pageblock_nr_pages has not already been setup */
  3393. if (pageblock_order)
  3394. return;
  3395. /*
  3396. * Assume the largest contiguous order of interest is a huge page.
  3397. * This value may be variable depending on boot parameters on IA64
  3398. */
  3399. pageblock_order = order;
  3400. }
  3401. #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
  3402. /*
  3403. * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
  3404. * and pageblock_default_order() are unused as pageblock_order is set
  3405. * at compile-time. See include/linux/pageblock-flags.h for the values of
  3406. * pageblock_order based on the kernel config
  3407. */
  3408. static inline int pageblock_default_order(unsigned int order)
  3409. {
  3410. return MAX_ORDER-1;
  3411. }
  3412. #define set_pageblock_order(x) do {} while (0)
  3413. #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
  3414. /*
  3415. * Set up the zone data structures:
  3416. * - mark all pages reserved
  3417. * - mark all memory queues empty
  3418. * - clear the memory bitmaps
  3419. */
  3420. static void __paginginit free_area_init_core(struct pglist_data *pgdat,
  3421. unsigned long *zones_size, unsigned long *zholes_size)
  3422. {
  3423. enum zone_type j;
  3424. int nid = pgdat->node_id;
  3425. unsigned long zone_start_pfn = pgdat->node_start_pfn;
  3426. int ret;
  3427. pgdat_resize_init(pgdat);
  3428. pgdat->nr_zones = 0;
  3429. init_waitqueue_head(&pgdat->kswapd_wait);
  3430. pgdat->kswapd_max_order = 0;
  3431. pgdat_page_cgroup_init(pgdat);
  3432. for (j = 0; j < MAX_NR_ZONES; j++) {
  3433. struct zone *zone = pgdat->node_zones + j;
  3434. unsigned long size, realsize, memmap_pages;
  3435. enum lru_list l;
  3436. size = zone_spanned_pages_in_node(nid, j, zones_size);
  3437. realsize = size - zone_absent_pages_in_node(nid, j,
  3438. zholes_size);
  3439. /*
  3440. * Adjust realsize so that it accounts for how much memory
  3441. * is used by this zone for memmap. This affects the watermark
  3442. * and per-cpu initialisations
  3443. */
  3444. memmap_pages =
  3445. PAGE_ALIGN(size * sizeof(struct page)) >> PAGE_SHIFT;
  3446. if (realsize >= memmap_pages) {
  3447. realsize -= memmap_pages;
  3448. if (memmap_pages)
  3449. printk(KERN_DEBUG
  3450. " %s zone: %lu pages used for memmap\n",
  3451. zone_names[j], memmap_pages);
  3452. } else
  3453. printk(KERN_WARNING
  3454. " %s zone: %lu pages exceeds realsize %lu\n",
  3455. zone_names[j], memmap_pages, realsize);
  3456. /* Account for reserved pages */
  3457. if (j == 0 && realsize > dma_reserve) {
  3458. realsize -= dma_reserve;
  3459. printk(KERN_DEBUG " %s zone: %lu pages reserved\n",
  3460. zone_names[0], dma_reserve);
  3461. }
  3462. if (!is_highmem_idx(j))
  3463. nr_kernel_pages += realsize;
  3464. nr_all_pages += realsize;
  3465. zone->spanned_pages = size;
  3466. zone->present_pages = realsize;
  3467. #ifdef CONFIG_NUMA
  3468. zone->node = nid;
  3469. zone->min_unmapped_pages = (realsize*sysctl_min_unmapped_ratio)
  3470. / 100;
  3471. zone->min_slab_pages = (realsize * sysctl_min_slab_ratio) / 100;
  3472. #endif
  3473. zone->name = zone_names[j];
  3474. spin_lock_init(&zone->lock);
  3475. spin_lock_init(&zone->lru_lock);
  3476. zone_seqlock_init(zone);
  3477. zone->zone_pgdat = pgdat;
  3478. zone->prev_priority = DEF_PRIORITY;
  3479. zone_pcp_init(zone);
  3480. for_each_lru(l) {
  3481. INIT_LIST_HEAD(&zone->lru[l].list);
  3482. zone->reclaim_stat.nr_saved_scan[l] = 0;
  3483. }
  3484. zone->reclaim_stat.recent_rotated[0] = 0;
  3485. zone->reclaim_stat.recent_rotated[1] = 0;
  3486. zone->reclaim_stat.recent_scanned[0] = 0;
  3487. zone->reclaim_stat.recent_scanned[1] = 0;
  3488. zap_zone_vm_stats(zone);
  3489. zone->flags = 0;
  3490. if (!size)
  3491. continue;
  3492. set_pageblock_order(pageblock_default_order());
  3493. setup_usemap(pgdat, zone, size);
  3494. ret = init_currently_empty_zone(zone, zone_start_pfn,
  3495. size, MEMMAP_EARLY);
  3496. BUG_ON(ret);
  3497. memmap_init(size, nid, j, zone_start_pfn);
  3498. zone_start_pfn += size;
  3499. }
  3500. }
  3501. static void __init_refok alloc_node_mem_map(struct pglist_data *pgdat)
  3502. {
  3503. /* Skip empty nodes */
  3504. if (!pgdat->node_spanned_pages)
  3505. return;
  3506. #ifdef CONFIG_FLAT_NODE_MEM_MAP
  3507. /* ia64 gets its own node_mem_map, before this, without bootmem */
  3508. if (!pgdat->node_mem_map) {
  3509. unsigned long size, start, end;
  3510. struct page *map;
  3511. /*
  3512. * The zone's endpoints aren't required to be MAX_ORDER
  3513. * aligned but the node_mem_map endpoints must be in order
  3514. * for the buddy allocator to function correctly.
  3515. */
  3516. start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
  3517. end = pgdat->node_start_pfn + pgdat->node_spanned_pages;
  3518. end = ALIGN(end, MAX_ORDER_NR_PAGES);
  3519. size = (end - start) * sizeof(struct page);
  3520. map = alloc_remap(pgdat->node_id, size);
  3521. if (!map)
  3522. map = alloc_bootmem_node(pgdat, size);
  3523. pgdat->node_mem_map = map + (pgdat->node_start_pfn - start);
  3524. }
  3525. #ifndef CONFIG_NEED_MULTIPLE_NODES
  3526. /*
  3527. * With no DISCONTIG, the global mem_map is just set as node 0's
  3528. */
  3529. if (pgdat == NODE_DATA(0)) {
  3530. mem_map = NODE_DATA(0)->node_mem_map;
  3531. #ifdef CONFIG_ARCH_POPULATES_NODE_MAP
  3532. if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
  3533. mem_map -= (pgdat->node_start_pfn - ARCH_PFN_OFFSET);
  3534. #endif /* CONFIG_ARCH_POPULATES_NODE_MAP */
  3535. }
  3536. #endif
  3537. #endif /* CONFIG_FLAT_NODE_MEM_MAP */
  3538. }
  3539. void __paginginit free_area_init_node(int nid, unsigned long *zones_size,
  3540. unsigned long node_start_pfn, unsigned long *zholes_size)
  3541. {
  3542. pg_data_t *pgdat = NODE_DATA(nid);
  3543. pgdat->node_id = nid;
  3544. pgdat->node_start_pfn = node_start_pfn;
  3545. calculate_node_totalpages(pgdat, zones_size, zholes_size);
  3546. alloc_node_mem_map(pgdat);
  3547. #ifdef CONFIG_FLAT_NODE_MEM_MAP
  3548. printk(KERN_DEBUG "free_area_init_node: node %d, pgdat %08lx, node_mem_map %08lx\n",
  3549. nid, (unsigned long)pgdat,
  3550. (unsigned long)pgdat->node_mem_map);
  3551. #endif
  3552. free_area_init_core(pgdat, zones_size, zholes_size);
  3553. }
  3554. #ifdef CONFIG_ARCH_POPULATES_NODE_MAP
  3555. #if MAX_NUMNODES > 1
  3556. /*
  3557. * Figure out the number of possible node ids.
  3558. */
  3559. static void __init setup_nr_node_ids(void)
  3560. {
  3561. unsigned int node;
  3562. unsigned int highest = 0;
  3563. for_each_node_mask(node, node_possible_map)
  3564. highest = node;
  3565. nr_node_ids = highest + 1;
  3566. }
  3567. #else
  3568. static inline void setup_nr_node_ids(void)
  3569. {
  3570. }
  3571. #endif
  3572. /**
  3573. * add_active_range - Register a range of PFNs backed by physical memory
  3574. * @nid: The node ID the range resides on
  3575. * @start_pfn: The start PFN of the available physical memory
  3576. * @end_pfn: The end PFN of the available physical memory
  3577. *
  3578. * These ranges are stored in an early_node_map[] and later used by
  3579. * free_area_init_nodes() to calculate zone sizes and holes. If the
  3580. * range spans a memory hole, it is up to the architecture to ensure
  3581. * the memory is not freed by the bootmem allocator. If possible
  3582. * the range being registered will be merged with existing ranges.
  3583. */
  3584. void __init add_active_range(unsigned int nid, unsigned long start_pfn,
  3585. unsigned long end_pfn)
  3586. {
  3587. int i;
  3588. mminit_dprintk(MMINIT_TRACE, "memory_register",
  3589. "Entering add_active_range(%d, %#lx, %#lx) "
  3590. "%d entries of %d used\n",
  3591. nid, start_pfn, end_pfn,
  3592. nr_nodemap_entries, MAX_ACTIVE_REGIONS);
  3593. mminit_validate_memmodel_limits(&start_pfn, &end_pfn);
  3594. /* Merge with existing active regions if possible */
  3595. for (i = 0; i < nr_nodemap_entries; i++) {
  3596. if (early_node_map[i].nid != nid)
  3597. continue;
  3598. /* Skip if an existing region covers this new one */
  3599. if (start_pfn >= early_node_map[i].start_pfn &&
  3600. end_pfn <= early_node_map[i].end_pfn)
  3601. return;
  3602. /* Merge forward if suitable */
  3603. if (start_pfn <= early_node_map[i].end_pfn &&
  3604. end_pfn > early_node_map[i].end_pfn) {
  3605. early_node_map[i].end_pfn = end_pfn;
  3606. return;
  3607. }
  3608. /* Merge backward if suitable */
  3609. if (start_pfn < early_node_map[i].start_pfn &&
  3610. end_pfn >= early_node_map[i].start_pfn) {
  3611. early_node_map[i].start_pfn = start_pfn;
  3612. return;
  3613. }
  3614. }
  3615. /* Check that early_node_map is large enough */
  3616. if (i >= MAX_ACTIVE_REGIONS) {
  3617. printk(KERN_CRIT "More than %d memory regions, truncating\n",
  3618. MAX_ACTIVE_REGIONS);
  3619. return;
  3620. }
  3621. early_node_map[i].nid = nid;
  3622. early_node_map[i].start_pfn = start_pfn;
  3623. early_node_map[i].end_pfn = end_pfn;
  3624. nr_nodemap_entries = i + 1;
  3625. }
  3626. /**
  3627. * remove_active_range - Shrink an existing registered range of PFNs
  3628. * @nid: The node id the range is on that should be shrunk
  3629. * @start_pfn: The new PFN of the range
  3630. * @end_pfn: The new PFN of the range
  3631. *
  3632. * i386 with NUMA use alloc_remap() to store a node_mem_map on a local node.
  3633. * The map is kept near the end physical page range that has already been
  3634. * registered. This function allows an arch to shrink an existing registered
  3635. * range.
  3636. */
  3637. void __init remove_active_range(unsigned int nid, unsigned long start_pfn,
  3638. unsigned long end_pfn)
  3639. {
  3640. int i, j;
  3641. int removed = 0;
  3642. printk(KERN_DEBUG "remove_active_range (%d, %lu, %lu)\n",
  3643. nid, start_pfn, end_pfn);
  3644. /* Find the old active region end and shrink */
  3645. for_each_active_range_index_in_nid(i, nid) {
  3646. if (early_node_map[i].start_pfn >= start_pfn &&
  3647. early_node_map[i].end_pfn <= end_pfn) {
  3648. /* clear it */
  3649. early_node_map[i].start_pfn = 0;
  3650. early_node_map[i].end_pfn = 0;
  3651. removed = 1;
  3652. continue;
  3653. }
  3654. if (early_node_map[i].start_pfn < start_pfn &&
  3655. early_node_map[i].end_pfn > start_pfn) {
  3656. unsigned long temp_end_pfn = early_node_map[i].end_pfn;
  3657. early_node_map[i].end_pfn = start_pfn;
  3658. if (temp_end_pfn > end_pfn)
  3659. add_active_range(nid, end_pfn, temp_end_pfn);
  3660. continue;
  3661. }
  3662. if (early_node_map[i].start_pfn >= start_pfn &&
  3663. early_node_map[i].end_pfn > end_pfn &&
  3664. early_node_map[i].start_pfn < end_pfn) {
  3665. early_node_map[i].start_pfn = end_pfn;
  3666. continue;
  3667. }
  3668. }
  3669. if (!removed)
  3670. return;
  3671. /* remove the blank ones */
  3672. for (i = nr_nodemap_entries - 1; i > 0; i--) {
  3673. if (early_node_map[i].nid != nid)
  3674. continue;
  3675. if (early_node_map[i].end_pfn)
  3676. continue;
  3677. /* we found it, get rid of it */
  3678. for (j = i; j < nr_nodemap_entries - 1; j++)
  3679. memcpy(&early_node_map[j], &early_node_map[j+1],
  3680. sizeof(early_node_map[j]));
  3681. j = nr_nodemap_entries - 1;
  3682. memset(&early_node_map[j], 0, sizeof(early_node_map[j]));
  3683. nr_nodemap_entries--;
  3684. }
  3685. }
  3686. /**
  3687. * remove_all_active_ranges - Remove all currently registered regions
  3688. *
  3689. * During discovery, it may be found that a table like SRAT is invalid
  3690. * and an alternative discovery method must be used. This function removes
  3691. * all currently registered regions.
  3692. */
  3693. void __init remove_all_active_ranges(void)
  3694. {
  3695. memset(early_node_map, 0, sizeof(early_node_map));
  3696. nr_nodemap_entries = 0;
  3697. }
  3698. /* Compare two active node_active_regions */
  3699. static int __init cmp_node_active_region(const void *a, const void *b)
  3700. {
  3701. struct node_active_region *arange = (struct node_active_region *)a;
  3702. struct node_active_region *brange = (struct node_active_region *)b;
  3703. /* Done this way to avoid overflows */
  3704. if (arange->start_pfn > brange->start_pfn)
  3705. return 1;
  3706. if (arange->start_pfn < brange->start_pfn)
  3707. return -1;
  3708. return 0;
  3709. }
  3710. /* sort the node_map by start_pfn */
  3711. void __init sort_node_map(void)
  3712. {
  3713. sort(early_node_map, (size_t)nr_nodemap_entries,
  3714. sizeof(struct node_active_region),
  3715. cmp_node_active_region, NULL);
  3716. }
  3717. /* Find the lowest pfn for a node */
  3718. static unsigned long __init find_min_pfn_for_node(int nid)
  3719. {
  3720. int i;
  3721. unsigned long min_pfn = ULONG_MAX;
  3722. /* Assuming a sorted map, the first range found has the starting pfn */
  3723. for_each_active_range_index_in_nid(i, nid)
  3724. min_pfn = min(min_pfn, early_node_map[i].start_pfn);
  3725. if (min_pfn == ULONG_MAX) {
  3726. printk(KERN_WARNING
  3727. "Could not find start_pfn for node %d\n", nid);
  3728. return 0;
  3729. }
  3730. return min_pfn;
  3731. }
  3732. /**
  3733. * find_min_pfn_with_active_regions - Find the minimum PFN registered
  3734. *
  3735. * It returns the minimum PFN based on information provided via
  3736. * add_active_range().
  3737. */
  3738. unsigned long __init find_min_pfn_with_active_regions(void)
  3739. {
  3740. return find_min_pfn_for_node(MAX_NUMNODES);
  3741. }
  3742. /*
  3743. * early_calculate_totalpages()
  3744. * Sum pages in active regions for movable zone.
  3745. * Populate N_HIGH_MEMORY for calculating usable_nodes.
  3746. */
  3747. static unsigned long __init early_calculate_totalpages(void)
  3748. {
  3749. int i;
  3750. unsigned long totalpages = 0;
  3751. for (i = 0; i < nr_nodemap_entries; i++) {
  3752. unsigned long pages = early_node_map[i].end_pfn -
  3753. early_node_map[i].start_pfn;
  3754. totalpages += pages;
  3755. if (pages)
  3756. node_set_state(early_node_map[i].nid, N_HIGH_MEMORY);
  3757. }
  3758. return totalpages;
  3759. }
  3760. /*
  3761. * Find the PFN the Movable zone begins in each node. Kernel memory
  3762. * is spread evenly between nodes as long as the nodes have enough
  3763. * memory. When they don't, some nodes will have more kernelcore than
  3764. * others
  3765. */
  3766. static void __init find_zone_movable_pfns_for_nodes(unsigned long *movable_pfn)
  3767. {
  3768. int i, nid;
  3769. unsigned long usable_startpfn;
  3770. unsigned long kernelcore_node, kernelcore_remaining;
  3771. /* save the state before borrow the nodemask */
  3772. nodemask_t saved_node_state = node_states[N_HIGH_MEMORY];
  3773. unsigned long totalpages = early_calculate_totalpages();
  3774. int usable_nodes = nodes_weight(node_states[N_HIGH_MEMORY]);
  3775. /*
  3776. * If movablecore was specified, calculate what size of
  3777. * kernelcore that corresponds so that memory usable for
  3778. * any allocation type is evenly spread. If both kernelcore
  3779. * and movablecore are specified, then the value of kernelcore
  3780. * will be used for required_kernelcore if it's greater than
  3781. * what movablecore would have allowed.
  3782. */
  3783. if (required_movablecore) {
  3784. unsigned long corepages;
  3785. /*
  3786. * Round-up so that ZONE_MOVABLE is at least as large as what
  3787. * was requested by the user
  3788. */
  3789. required_movablecore =
  3790. roundup(required_movablecore, MAX_ORDER_NR_PAGES);
  3791. corepages = totalpages - required_movablecore;
  3792. required_kernelcore = max(required_kernelcore, corepages);
  3793. }
  3794. /* If kernelcore was not specified, there is no ZONE_MOVABLE */
  3795. if (!required_kernelcore)
  3796. goto out;
  3797. /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
  3798. find_usable_zone_for_movable();
  3799. usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
  3800. restart:
  3801. /* Spread kernelcore memory as evenly as possible throughout nodes */
  3802. kernelcore_node = required_kernelcore / usable_nodes;
  3803. for_each_node_state(nid, N_HIGH_MEMORY) {
  3804. /*
  3805. * Recalculate kernelcore_node if the division per node
  3806. * now exceeds what is necessary to satisfy the requested
  3807. * amount of memory for the kernel
  3808. */
  3809. if (required_kernelcore < kernelcore_node)
  3810. kernelcore_node = required_kernelcore / usable_nodes;
  3811. /*
  3812. * As the map is walked, we track how much memory is usable
  3813. * by the kernel using kernelcore_remaining. When it is
  3814. * 0, the rest of the node is usable by ZONE_MOVABLE
  3815. */
  3816. kernelcore_remaining = kernelcore_node;
  3817. /* Go through each range of PFNs within this node */
  3818. for_each_active_range_index_in_nid(i, nid) {
  3819. unsigned long start_pfn, end_pfn;
  3820. unsigned long size_pages;
  3821. start_pfn = max(early_node_map[i].start_pfn,
  3822. zone_movable_pfn[nid]);
  3823. end_pfn = early_node_map[i].end_pfn;
  3824. if (start_pfn >= end_pfn)
  3825. continue;
  3826. /* Account for what is only usable for kernelcore */
  3827. if (start_pfn < usable_startpfn) {
  3828. unsigned long kernel_pages;
  3829. kernel_pages = min(end_pfn, usable_startpfn)
  3830. - start_pfn;
  3831. kernelcore_remaining -= min(kernel_pages,
  3832. kernelcore_remaining);
  3833. required_kernelcore -= min(kernel_pages,
  3834. required_kernelcore);
  3835. /* Continue if range is now fully accounted */
  3836. if (end_pfn <= usable_startpfn) {
  3837. /*
  3838. * Push zone_movable_pfn to the end so
  3839. * that if we have to rebalance
  3840. * kernelcore across nodes, we will
  3841. * not double account here
  3842. */
  3843. zone_movable_pfn[nid] = end_pfn;
  3844. continue;
  3845. }
  3846. start_pfn = usable_startpfn;
  3847. }
  3848. /*
  3849. * The usable PFN range for ZONE_MOVABLE is from
  3850. * start_pfn->end_pfn. Calculate size_pages as the
  3851. * number of pages used as kernelcore
  3852. */
  3853. size_pages = end_pfn - start_pfn;
  3854. if (size_pages > kernelcore_remaining)
  3855. size_pages = kernelcore_remaining;
  3856. zone_movable_pfn[nid] = start_pfn + size_pages;
  3857. /*
  3858. * Some kernelcore has been met, update counts and
  3859. * break if the kernelcore for this node has been
  3860. * satisified
  3861. */
  3862. required_kernelcore -= min(required_kernelcore,
  3863. size_pages);
  3864. kernelcore_remaining -= size_pages;
  3865. if (!kernelcore_remaining)
  3866. break;
  3867. }
  3868. }
  3869. /*
  3870. * If there is still required_kernelcore, we do another pass with one
  3871. * less node in the count. This will push zone_movable_pfn[nid] further
  3872. * along on the nodes that still have memory until kernelcore is
  3873. * satisified
  3874. */
  3875. usable_nodes--;
  3876. if (usable_nodes && required_kernelcore > usable_nodes)
  3877. goto restart;
  3878. /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
  3879. for (nid = 0; nid < MAX_NUMNODES; nid++)
  3880. zone_movable_pfn[nid] =
  3881. roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
  3882. out:
  3883. /* restore the node_state */
  3884. node_states[N_HIGH_MEMORY] = saved_node_state;
  3885. }
  3886. /* Any regular memory on that node ? */
  3887. static void check_for_regular_memory(pg_data_t *pgdat)
  3888. {
  3889. #ifdef CONFIG_HIGHMEM
  3890. enum zone_type zone_type;
  3891. for (zone_type = 0; zone_type <= ZONE_NORMAL; zone_type++) {
  3892. struct zone *zone = &pgdat->node_zones[zone_type];
  3893. if (zone->present_pages)
  3894. node_set_state(zone_to_nid(zone), N_NORMAL_MEMORY);
  3895. }
  3896. #endif
  3897. }
  3898. /**
  3899. * free_area_init_nodes - Initialise all pg_data_t and zone data
  3900. * @max_zone_pfn: an array of max PFNs for each zone
  3901. *
  3902. * This will call free_area_init_node() for each active node in the system.
  3903. * Using the page ranges provided by add_active_range(), the size of each
  3904. * zone in each node and their holes is calculated. If the maximum PFN
  3905. * between two adjacent zones match, it is assumed that the zone is empty.
  3906. * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
  3907. * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
  3908. * starts where the previous one ended. For example, ZONE_DMA32 starts
  3909. * at arch_max_dma_pfn.
  3910. */
  3911. void __init free_area_init_nodes(unsigned long *max_zone_pfn)
  3912. {
  3913. unsigned long nid;
  3914. int i;
  3915. /* Sort early_node_map as initialisation assumes it is sorted */
  3916. sort_node_map();
  3917. /* Record where the zone boundaries are */
  3918. memset(arch_zone_lowest_possible_pfn, 0,
  3919. sizeof(arch_zone_lowest_possible_pfn));
  3920. memset(arch_zone_highest_possible_pfn, 0,
  3921. sizeof(arch_zone_highest_possible_pfn));
  3922. arch_zone_lowest_possible_pfn[0] = find_min_pfn_with_active_regions();
  3923. arch_zone_highest_possible_pfn[0] = max_zone_pfn[0];
  3924. for (i = 1; i < MAX_NR_ZONES; i++) {
  3925. if (i == ZONE_MOVABLE)
  3926. continue;
  3927. arch_zone_lowest_possible_pfn[i] =
  3928. arch_zone_highest_possible_pfn[i-1];
  3929. arch_zone_highest_possible_pfn[i] =
  3930. max(max_zone_pfn[i], arch_zone_lowest_possible_pfn[i]);
  3931. }
  3932. arch_zone_lowest_possible_pfn[ZONE_MOVABLE] = 0;
  3933. arch_zone_highest_possible_pfn[ZONE_MOVABLE] = 0;
  3934. /* Find the PFNs that ZONE_MOVABLE begins at in each node */
  3935. memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
  3936. find_zone_movable_pfns_for_nodes(zone_movable_pfn);
  3937. /* Print out the zone ranges */
  3938. printk("Zone PFN ranges:\n");
  3939. for (i = 0; i < MAX_NR_ZONES; i++) {
  3940. if (i == ZONE_MOVABLE)
  3941. continue;
  3942. printk(" %-8s ", zone_names[i]);
  3943. if (arch_zone_lowest_possible_pfn[i] ==
  3944. arch_zone_highest_possible_pfn[i])
  3945. printk("empty\n");
  3946. else
  3947. printk("%0#10lx -> %0#10lx\n",
  3948. arch_zone_lowest_possible_pfn[i],
  3949. arch_zone_highest_possible_pfn[i]);
  3950. }
  3951. /* Print out the PFNs ZONE_MOVABLE begins at in each node */
  3952. printk("Movable zone start PFN for each node\n");
  3953. for (i = 0; i < MAX_NUMNODES; i++) {
  3954. if (zone_movable_pfn[i])
  3955. printk(" Node %d: %lu\n", i, zone_movable_pfn[i]);
  3956. }
  3957. /* Print out the early_node_map[] */
  3958. printk("early_node_map[%d] active PFN ranges\n", nr_nodemap_entries);
  3959. for (i = 0; i < nr_nodemap_entries; i++)
  3960. printk(" %3d: %0#10lx -> %0#10lx\n", early_node_map[i].nid,
  3961. early_node_map[i].start_pfn,
  3962. early_node_map[i].end_pfn);
  3963. /* Initialise every node */
  3964. mminit_verify_pageflags_layout();
  3965. setup_nr_node_ids();
  3966. for_each_online_node(nid) {
  3967. pg_data_t *pgdat = NODE_DATA(nid);
  3968. free_area_init_node(nid, NULL,
  3969. find_min_pfn_for_node(nid), NULL);
  3970. /* Any memory on that node */
  3971. if (pgdat->node_present_pages)
  3972. node_set_state(nid, N_HIGH_MEMORY);
  3973. check_for_regular_memory(pgdat);
  3974. }
  3975. }
  3976. static int __init cmdline_parse_core(char *p, unsigned long *core)
  3977. {
  3978. unsigned long long coremem;
  3979. if (!p)
  3980. return -EINVAL;
  3981. coremem = memparse(p, &p);
  3982. *core = coremem >> PAGE_SHIFT;
  3983. /* Paranoid check that UL is enough for the coremem value */
  3984. WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
  3985. return 0;
  3986. }
  3987. /*
  3988. * kernelcore=size sets the amount of memory for use for allocations that
  3989. * cannot be reclaimed or migrated.
  3990. */
  3991. static int __init cmdline_parse_kernelcore(char *p)
  3992. {
  3993. return cmdline_parse_core(p, &required_kernelcore);
  3994. }
  3995. /*
  3996. * movablecore=size sets the amount of memory for use for allocations that
  3997. * can be reclaimed or migrated.
  3998. */
  3999. static int __init cmdline_parse_movablecore(char *p)
  4000. {
  4001. return cmdline_parse_core(p, &required_movablecore);
  4002. }
  4003. early_param("kernelcore", cmdline_parse_kernelcore);
  4004. early_param("movablecore", cmdline_parse_movablecore);
  4005. #endif /* CONFIG_ARCH_POPULATES_NODE_MAP */
  4006. /**
  4007. * set_dma_reserve - set the specified number of pages reserved in the first zone
  4008. * @new_dma_reserve: The number of pages to mark reserved
  4009. *
  4010. * The per-cpu batchsize and zone watermarks are determined by present_pages.
  4011. * In the DMA zone, a significant percentage may be consumed by kernel image
  4012. * and other unfreeable allocations which can skew the watermarks badly. This
  4013. * function may optionally be used to account for unfreeable pages in the
  4014. * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
  4015. * smaller per-cpu batchsize.
  4016. */
  4017. void __init set_dma_reserve(unsigned long new_dma_reserve)
  4018. {
  4019. dma_reserve = new_dma_reserve;
  4020. }
  4021. #ifndef CONFIG_NEED_MULTIPLE_NODES
  4022. struct pglist_data __refdata contig_page_data = {
  4023. #ifndef CONFIG_NO_BOOTMEM
  4024. .bdata = &bootmem_node_data[0]
  4025. #endif
  4026. };
  4027. EXPORT_SYMBOL(contig_page_data);
  4028. #endif
  4029. void __init free_area_init(unsigned long *zones_size)
  4030. {
  4031. free_area_init_node(0, zones_size,
  4032. __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL);
  4033. }
  4034. static int page_alloc_cpu_notify(struct notifier_block *self,
  4035. unsigned long action, void *hcpu)
  4036. {
  4037. int cpu = (unsigned long)hcpu;
  4038. if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
  4039. drain_pages(cpu);
  4040. /*
  4041. * Spill the event counters of the dead processor
  4042. * into the current processors event counters.
  4043. * This artificially elevates the count of the current
  4044. * processor.
  4045. */
  4046. vm_events_fold_cpu(cpu);
  4047. /*
  4048. * Zero the differential counters of the dead processor
  4049. * so that the vm statistics are consistent.
  4050. *
  4051. * This is only okay since the processor is dead and cannot
  4052. * race with what we are doing.
  4053. */
  4054. refresh_cpu_vm_stats(cpu);
  4055. }
  4056. return NOTIFY_OK;
  4057. }
  4058. void __init page_alloc_init(void)
  4059. {
  4060. hotcpu_notifier(page_alloc_cpu_notify, 0);
  4061. }
  4062. /*
  4063. * calculate_totalreserve_pages - called when sysctl_lower_zone_reserve_ratio
  4064. * or min_free_kbytes changes.
  4065. */
  4066. static void calculate_totalreserve_pages(void)
  4067. {
  4068. struct pglist_data *pgdat;
  4069. unsigned long reserve_pages = 0;
  4070. enum zone_type i, j;
  4071. for_each_online_pgdat(pgdat) {
  4072. for (i = 0; i < MAX_NR_ZONES; i++) {
  4073. struct zone *zone = pgdat->node_zones + i;
  4074. unsigned long max = 0;
  4075. /* Find valid and maximum lowmem_reserve in the zone */
  4076. for (j = i; j < MAX_NR_ZONES; j++) {
  4077. if (zone->lowmem_reserve[j] > max)
  4078. max = zone->lowmem_reserve[j];
  4079. }
  4080. /* we treat the high watermark as reserved pages. */
  4081. max += high_wmark_pages(zone);
  4082. if (max > zone->present_pages)
  4083. max = zone->present_pages;
  4084. reserve_pages += max;
  4085. }
  4086. }
  4087. totalreserve_pages = reserve_pages;
  4088. }
  4089. /*
  4090. * setup_per_zone_lowmem_reserve - called whenever
  4091. * sysctl_lower_zone_reserve_ratio changes. Ensures that each zone
  4092. * has a correct pages reserved value, so an adequate number of
  4093. * pages are left in the zone after a successful __alloc_pages().
  4094. */
  4095. static void setup_per_zone_lowmem_reserve(void)
  4096. {
  4097. struct pglist_data *pgdat;
  4098. enum zone_type j, idx;
  4099. for_each_online_pgdat(pgdat) {
  4100. for (j = 0; j < MAX_NR_ZONES; j++) {
  4101. struct zone *zone = pgdat->node_zones + j;
  4102. unsigned long present_pages = zone->present_pages;
  4103. zone->lowmem_reserve[j] = 0;
  4104. idx = j;
  4105. while (idx) {
  4106. struct zone *lower_zone;
  4107. idx--;
  4108. if (sysctl_lowmem_reserve_ratio[idx] < 1)
  4109. sysctl_lowmem_reserve_ratio[idx] = 1;
  4110. lower_zone = pgdat->node_zones + idx;
  4111. lower_zone->lowmem_reserve[j] = present_pages /
  4112. sysctl_lowmem_reserve_ratio[idx];
  4113. present_pages += lower_zone->present_pages;
  4114. }
  4115. }
  4116. }
  4117. /* update totalreserve_pages */
  4118. calculate_totalreserve_pages();
  4119. }
  4120. /**
  4121. * setup_per_zone_wmarks - called when min_free_kbytes changes
  4122. * or when memory is hot-{added|removed}
  4123. *
  4124. * Ensures that the watermark[min,low,high] values for each zone are set
  4125. * correctly with respect to min_free_kbytes.
  4126. */
  4127. void setup_per_zone_wmarks(void)
  4128. {
  4129. unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
  4130. unsigned long lowmem_pages = 0;
  4131. struct zone *zone;
  4132. unsigned long flags;
  4133. /* Calculate total number of !ZONE_HIGHMEM pages */
  4134. for_each_zone(zone) {
  4135. if (!is_highmem(zone))
  4136. lowmem_pages += zone->present_pages;
  4137. }
  4138. for_each_zone(zone) {
  4139. u64 tmp;
  4140. spin_lock_irqsave(&zone->lock, flags);
  4141. tmp = (u64)pages_min * zone->present_pages;
  4142. do_div(tmp, lowmem_pages);
  4143. if (is_highmem(zone)) {
  4144. /*
  4145. * __GFP_HIGH and PF_MEMALLOC allocations usually don't
  4146. * need highmem pages, so cap pages_min to a small
  4147. * value here.
  4148. *
  4149. * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
  4150. * deltas controls asynch page reclaim, and so should
  4151. * not be capped for highmem.
  4152. */
  4153. int min_pages;
  4154. min_pages = zone->present_pages / 1024;
  4155. if (min_pages < SWAP_CLUSTER_MAX)
  4156. min_pages = SWAP_CLUSTER_MAX;
  4157. if (min_pages > 128)
  4158. min_pages = 128;
  4159. zone->watermark[WMARK_MIN] = min_pages;
  4160. } else {
  4161. /*
  4162. * If it's a lowmem zone, reserve a number of pages
  4163. * proportionate to the zone's size.
  4164. */
  4165. zone->watermark[WMARK_MIN] = tmp;
  4166. }
  4167. zone->watermark[WMARK_LOW] = min_wmark_pages(zone) + (tmp >> 2);
  4168. zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + (tmp >> 1);
  4169. setup_zone_migrate_reserve(zone);
  4170. spin_unlock_irqrestore(&zone->lock, flags);
  4171. }
  4172. /* update totalreserve_pages */
  4173. calculate_totalreserve_pages();
  4174. }
  4175. /*
  4176. * The inactive anon list should be small enough that the VM never has to
  4177. * do too much work, but large enough that each inactive page has a chance
  4178. * to be referenced again before it is swapped out.
  4179. *
  4180. * The inactive_anon ratio is the target ratio of ACTIVE_ANON to
  4181. * INACTIVE_ANON pages on this zone's LRU, maintained by the
  4182. * pageout code. A zone->inactive_ratio of 3 means 3:1 or 25% of
  4183. * the anonymous pages are kept on the inactive list.
  4184. *
  4185. * total target max
  4186. * memory ratio inactive anon
  4187. * -------------------------------------
  4188. * 10MB 1 5MB
  4189. * 100MB 1 50MB
  4190. * 1GB 3 250MB
  4191. * 10GB 10 0.9GB
  4192. * 100GB 31 3GB
  4193. * 1TB 101 10GB
  4194. * 10TB 320 32GB
  4195. */
  4196. void calculate_zone_inactive_ratio(struct zone *zone)
  4197. {
  4198. unsigned int gb, ratio;
  4199. /* Zone size in gigabytes */
  4200. gb = zone->present_pages >> (30 - PAGE_SHIFT);
  4201. if (gb)
  4202. ratio = int_sqrt(10 * gb);
  4203. else
  4204. ratio = 1;
  4205. zone->inactive_ratio = ratio;
  4206. }
  4207. static void __init setup_per_zone_inactive_ratio(void)
  4208. {
  4209. struct zone *zone;
  4210. for_each_zone(zone)
  4211. calculate_zone_inactive_ratio(zone);
  4212. }
  4213. /*
  4214. * Initialise min_free_kbytes.
  4215. *
  4216. * For small machines we want it small (128k min). For large machines
  4217. * we want it large (64MB max). But it is not linear, because network
  4218. * bandwidth does not increase linearly with machine size. We use
  4219. *
  4220. * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy:
  4221. * min_free_kbytes = sqrt(lowmem_kbytes * 16)
  4222. *
  4223. * which yields
  4224. *
  4225. * 16MB: 512k
  4226. * 32MB: 724k
  4227. * 64MB: 1024k
  4228. * 128MB: 1448k
  4229. * 256MB: 2048k
  4230. * 512MB: 2896k
  4231. * 1024MB: 4096k
  4232. * 2048MB: 5792k
  4233. * 4096MB: 8192k
  4234. * 8192MB: 11584k
  4235. * 16384MB: 16384k
  4236. */
  4237. static int __init init_per_zone_wmark_min(void)
  4238. {
  4239. unsigned long lowmem_kbytes;
  4240. lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
  4241. min_free_kbytes = int_sqrt(lowmem_kbytes * 16);
  4242. if (min_free_kbytes < 128)
  4243. min_free_kbytes = 128;
  4244. if (min_free_kbytes > 65536)
  4245. min_free_kbytes = 65536;
  4246. setup_per_zone_wmarks();
  4247. setup_per_zone_lowmem_reserve();
  4248. setup_per_zone_inactive_ratio();
  4249. return 0;
  4250. }
  4251. module_init(init_per_zone_wmark_min)
  4252. /*
  4253. * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so
  4254. * that we can call two helper functions whenever min_free_kbytes
  4255. * changes.
  4256. */
  4257. int min_free_kbytes_sysctl_handler(ctl_table *table, int write,
  4258. void __user *buffer, size_t *length, loff_t *ppos)
  4259. {
  4260. proc_dointvec(table, write, buffer, length, ppos);
  4261. if (write)
  4262. setup_per_zone_wmarks();
  4263. return 0;
  4264. }
  4265. #ifdef CONFIG_NUMA
  4266. int sysctl_min_unmapped_ratio_sysctl_handler(ctl_table *table, int write,
  4267. void __user *buffer, size_t *length, loff_t *ppos)
  4268. {
  4269. struct zone *zone;
  4270. int rc;
  4271. rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
  4272. if (rc)
  4273. return rc;
  4274. for_each_zone(zone)
  4275. zone->min_unmapped_pages = (zone->present_pages *
  4276. sysctl_min_unmapped_ratio) / 100;
  4277. return 0;
  4278. }
  4279. int sysctl_min_slab_ratio_sysctl_handler(ctl_table *table, int write,
  4280. void __user *buffer, size_t *length, loff_t *ppos)
  4281. {
  4282. struct zone *zone;
  4283. int rc;
  4284. rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
  4285. if (rc)
  4286. return rc;
  4287. for_each_zone(zone)
  4288. zone->min_slab_pages = (zone->present_pages *
  4289. sysctl_min_slab_ratio) / 100;
  4290. return 0;
  4291. }
  4292. #endif
  4293. /*
  4294. * lowmem_reserve_ratio_sysctl_handler - just a wrapper around
  4295. * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve()
  4296. * whenever sysctl_lowmem_reserve_ratio changes.
  4297. *
  4298. * The reserve ratio obviously has absolutely no relation with the
  4299. * minimum watermarks. The lowmem reserve ratio can only make sense
  4300. * if in function of the boot time zone sizes.
  4301. */
  4302. int lowmem_reserve_ratio_sysctl_handler(ctl_table *table, int write,
  4303. void __user *buffer, size_t *length, loff_t *ppos)
  4304. {
  4305. proc_dointvec_minmax(table, write, buffer, length, ppos);
  4306. setup_per_zone_lowmem_reserve();
  4307. return 0;
  4308. }
  4309. /*
  4310. * percpu_pagelist_fraction - changes the pcp->high for each zone on each
  4311. * cpu. It is the fraction of total pages in each zone that a hot per cpu pagelist
  4312. * can have before it gets flushed back to buddy allocator.
  4313. */
  4314. int percpu_pagelist_fraction_sysctl_handler(ctl_table *table, int write,
  4315. void __user *buffer, size_t *length, loff_t *ppos)
  4316. {
  4317. struct zone *zone;
  4318. unsigned int cpu;
  4319. int ret;
  4320. ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
  4321. if (!write || (ret == -EINVAL))
  4322. return ret;
  4323. for_each_populated_zone(zone) {
  4324. for_each_possible_cpu(cpu) {
  4325. unsigned long high;
  4326. high = zone->present_pages / percpu_pagelist_fraction;
  4327. setup_pagelist_highmark(
  4328. per_cpu_ptr(zone->pageset, cpu), high);
  4329. }
  4330. }
  4331. return 0;
  4332. }
  4333. int hashdist = HASHDIST_DEFAULT;
  4334. #ifdef CONFIG_NUMA
  4335. static int __init set_hashdist(char *str)
  4336. {
  4337. if (!str)
  4338. return 0;
  4339. hashdist = simple_strtoul(str, &str, 0);
  4340. return 1;
  4341. }
  4342. __setup("hashdist=", set_hashdist);
  4343. #endif
  4344. /*
  4345. * allocate a large system hash table from bootmem
  4346. * - it is assumed that the hash table must contain an exact power-of-2
  4347. * quantity of entries
  4348. * - limit is the number of hash buckets, not the total allocation size
  4349. */
  4350. void *__init alloc_large_system_hash(const char *tablename,
  4351. unsigned long bucketsize,
  4352. unsigned long numentries,
  4353. int scale,
  4354. int flags,
  4355. unsigned int *_hash_shift,
  4356. unsigned int *_hash_mask,
  4357. unsigned long limit)
  4358. {
  4359. unsigned long long max = limit;
  4360. unsigned long log2qty, size;
  4361. void *table = NULL;
  4362. /* allow the kernel cmdline to have a say */
  4363. if (!numentries) {
  4364. /* round applicable memory size up to nearest megabyte */
  4365. numentries = nr_kernel_pages;
  4366. numentries += (1UL << (20 - PAGE_SHIFT)) - 1;
  4367. numentries >>= 20 - PAGE_SHIFT;
  4368. numentries <<= 20 - PAGE_SHIFT;
  4369. /* limit to 1 bucket per 2^scale bytes of low memory */
  4370. if (scale > PAGE_SHIFT)
  4371. numentries >>= (scale - PAGE_SHIFT);
  4372. else
  4373. numentries <<= (PAGE_SHIFT - scale);
  4374. /* Make sure we've got at least a 0-order allocation.. */
  4375. if (unlikely(flags & HASH_SMALL)) {
  4376. /* Makes no sense without HASH_EARLY */
  4377. WARN_ON(!(flags & HASH_EARLY));
  4378. if (!(numentries >> *_hash_shift)) {
  4379. numentries = 1UL << *_hash_shift;
  4380. BUG_ON(!numentries);
  4381. }
  4382. } else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
  4383. numentries = PAGE_SIZE / bucketsize;
  4384. }
  4385. numentries = roundup_pow_of_two(numentries);
  4386. /* limit allocation size to 1/16 total memory by default */
  4387. if (max == 0) {
  4388. max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
  4389. do_div(max, bucketsize);
  4390. }
  4391. if (numentries > max)
  4392. numentries = max;
  4393. log2qty = ilog2(numentries);
  4394. do {
  4395. size = bucketsize << log2qty;
  4396. if (flags & HASH_EARLY)
  4397. table = alloc_bootmem_nopanic(size);
  4398. else if (hashdist)
  4399. table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL);
  4400. else {
  4401. /*
  4402. * If bucketsize is not a power-of-two, we may free
  4403. * some pages at the end of hash table which
  4404. * alloc_pages_exact() automatically does
  4405. */
  4406. if (get_order(size) < MAX_ORDER) {
  4407. table = alloc_pages_exact(size, GFP_ATOMIC);
  4408. kmemleak_alloc(table, size, 1, GFP_ATOMIC);
  4409. }
  4410. }
  4411. } while (!table && size > PAGE_SIZE && --log2qty);
  4412. if (!table)
  4413. panic("Failed to allocate %s hash table\n", tablename);
  4414. printk(KERN_INFO "%s hash table entries: %d (order: %d, %lu bytes)\n",
  4415. tablename,
  4416. (1U << log2qty),
  4417. ilog2(size) - PAGE_SHIFT,
  4418. size);
  4419. if (_hash_shift)
  4420. *_hash_shift = log2qty;
  4421. if (_hash_mask)
  4422. *_hash_mask = (1 << log2qty) - 1;
  4423. return table;
  4424. }
  4425. /* Return a pointer to the bitmap storing bits affecting a block of pages */
  4426. static inline unsigned long *get_pageblock_bitmap(struct zone *zone,
  4427. unsigned long pfn)
  4428. {
  4429. #ifdef CONFIG_SPARSEMEM
  4430. return __pfn_to_section(pfn)->pageblock_flags;
  4431. #else
  4432. return zone->pageblock_flags;
  4433. #endif /* CONFIG_SPARSEMEM */
  4434. }
  4435. static inline int pfn_to_bitidx(struct zone *zone, unsigned long pfn)
  4436. {
  4437. #ifdef CONFIG_SPARSEMEM
  4438. pfn &= (PAGES_PER_SECTION-1);
  4439. return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
  4440. #else
  4441. pfn = pfn - zone->zone_start_pfn;
  4442. return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
  4443. #endif /* CONFIG_SPARSEMEM */
  4444. }
  4445. /**
  4446. * get_pageblock_flags_group - Return the requested group of flags for the pageblock_nr_pages block of pages
  4447. * @page: The page within the block of interest
  4448. * @start_bitidx: The first bit of interest to retrieve
  4449. * @end_bitidx: The last bit of interest
  4450. * returns pageblock_bits flags
  4451. */
  4452. unsigned long get_pageblock_flags_group(struct page *page,
  4453. int start_bitidx, int end_bitidx)
  4454. {
  4455. struct zone *zone;
  4456. unsigned long *bitmap;
  4457. unsigned long pfn, bitidx;
  4458. unsigned long flags = 0;
  4459. unsigned long value = 1;
  4460. zone = page_zone(page);
  4461. pfn = page_to_pfn(page);
  4462. bitmap = get_pageblock_bitmap(zone, pfn);
  4463. bitidx = pfn_to_bitidx(zone, pfn);
  4464. for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1)
  4465. if (test_bit(bitidx + start_bitidx, bitmap))
  4466. flags |= value;
  4467. return flags;
  4468. }
  4469. /**
  4470. * set_pageblock_flags_group - Set the requested group of flags for a pageblock_nr_pages block of pages
  4471. * @page: The page within the block of interest
  4472. * @start_bitidx: The first bit of interest
  4473. * @end_bitidx: The last bit of interest
  4474. * @flags: The flags to set
  4475. */
  4476. void set_pageblock_flags_group(struct page *page, unsigned long flags,
  4477. int start_bitidx, int end_bitidx)
  4478. {
  4479. struct zone *zone;
  4480. unsigned long *bitmap;
  4481. unsigned long pfn, bitidx;
  4482. unsigned long value = 1;
  4483. zone = page_zone(page);
  4484. pfn = page_to_pfn(page);
  4485. bitmap = get_pageblock_bitmap(zone, pfn);
  4486. bitidx = pfn_to_bitidx(zone, pfn);
  4487. VM_BUG_ON(pfn < zone->zone_start_pfn);
  4488. VM_BUG_ON(pfn >= zone->zone_start_pfn + zone->spanned_pages);
  4489. for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1)
  4490. if (flags & value)
  4491. __set_bit(bitidx + start_bitidx, bitmap);
  4492. else
  4493. __clear_bit(bitidx + start_bitidx, bitmap);
  4494. }
  4495. /*
  4496. * This is designed as sub function...plz see page_isolation.c also.
  4497. * set/clear page block's type to be ISOLATE.
  4498. * page allocater never alloc memory from ISOLATE block.
  4499. */
  4500. int set_migratetype_isolate(struct page *page)
  4501. {
  4502. struct zone *zone;
  4503. struct page *curr_page;
  4504. unsigned long flags, pfn, iter;
  4505. unsigned long immobile = 0;
  4506. struct memory_isolate_notify arg;
  4507. int notifier_ret;
  4508. int ret = -EBUSY;
  4509. int zone_idx;
  4510. zone = page_zone(page);
  4511. zone_idx = zone_idx(zone);
  4512. spin_lock_irqsave(&zone->lock, flags);
  4513. if (get_pageblock_migratetype(page) == MIGRATE_MOVABLE ||
  4514. zone_idx == ZONE_MOVABLE) {
  4515. ret = 0;
  4516. goto out;
  4517. }
  4518. pfn = page_to_pfn(page);
  4519. arg.start_pfn = pfn;
  4520. arg.nr_pages = pageblock_nr_pages;
  4521. arg.pages_found = 0;
  4522. /*
  4523. * It may be possible to isolate a pageblock even if the
  4524. * migratetype is not MIGRATE_MOVABLE. The memory isolation
  4525. * notifier chain is used by balloon drivers to return the
  4526. * number of pages in a range that are held by the balloon
  4527. * driver to shrink memory. If all the pages are accounted for
  4528. * by balloons, are free, or on the LRU, isolation can continue.
  4529. * Later, for example, when memory hotplug notifier runs, these
  4530. * pages reported as "can be isolated" should be isolated(freed)
  4531. * by the balloon driver through the memory notifier chain.
  4532. */
  4533. notifier_ret = memory_isolate_notify(MEM_ISOLATE_COUNT, &arg);
  4534. notifier_ret = notifier_to_errno(notifier_ret);
  4535. if (notifier_ret || !arg.pages_found)
  4536. goto out;
  4537. for (iter = pfn; iter < (pfn + pageblock_nr_pages); iter++) {
  4538. if (!pfn_valid_within(pfn))
  4539. continue;
  4540. curr_page = pfn_to_page(iter);
  4541. if (!page_count(curr_page) || PageLRU(curr_page))
  4542. continue;
  4543. immobile++;
  4544. }
  4545. if (arg.pages_found == immobile)
  4546. ret = 0;
  4547. out:
  4548. if (!ret) {
  4549. set_pageblock_migratetype(page, MIGRATE_ISOLATE);
  4550. move_freepages_block(zone, page, MIGRATE_ISOLATE);
  4551. }
  4552. spin_unlock_irqrestore(&zone->lock, flags);
  4553. if (!ret)
  4554. drain_all_pages();
  4555. return ret;
  4556. }
  4557. void unset_migratetype_isolate(struct page *page)
  4558. {
  4559. struct zone *zone;
  4560. unsigned long flags;
  4561. zone = page_zone(page);
  4562. spin_lock_irqsave(&zone->lock, flags);
  4563. if (get_pageblock_migratetype(page) != MIGRATE_ISOLATE)
  4564. goto out;
  4565. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  4566. move_freepages_block(zone, page, MIGRATE_MOVABLE);
  4567. out:
  4568. spin_unlock_irqrestore(&zone->lock, flags);
  4569. }
  4570. #ifdef CONFIG_MEMORY_HOTREMOVE
  4571. /*
  4572. * All pages in the range must be isolated before calling this.
  4573. */
  4574. void
  4575. __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
  4576. {
  4577. struct page *page;
  4578. struct zone *zone;
  4579. int order, i;
  4580. unsigned long pfn;
  4581. unsigned long flags;
  4582. /* find the first valid pfn */
  4583. for (pfn = start_pfn; pfn < end_pfn; pfn++)
  4584. if (pfn_valid(pfn))
  4585. break;
  4586. if (pfn == end_pfn)
  4587. return;
  4588. zone = page_zone(pfn_to_page(pfn));
  4589. spin_lock_irqsave(&zone->lock, flags);
  4590. pfn = start_pfn;
  4591. while (pfn < end_pfn) {
  4592. if (!pfn_valid(pfn)) {
  4593. pfn++;
  4594. continue;
  4595. }
  4596. page = pfn_to_page(pfn);
  4597. BUG_ON(page_count(page));
  4598. BUG_ON(!PageBuddy(page));
  4599. order = page_order(page);
  4600. #ifdef CONFIG_DEBUG_VM
  4601. printk(KERN_INFO "remove from free list %lx %d %lx\n",
  4602. pfn, 1 << order, end_pfn);
  4603. #endif
  4604. list_del(&page->lru);
  4605. rmv_page_order(page);
  4606. zone->free_area[order].nr_free--;
  4607. __mod_zone_page_state(zone, NR_FREE_PAGES,
  4608. - (1UL << order));
  4609. for (i = 0; i < (1 << order); i++)
  4610. SetPageReserved((page+i));
  4611. pfn += (1 << order);
  4612. }
  4613. spin_unlock_irqrestore(&zone->lock, flags);
  4614. }
  4615. #endif
  4616. #ifdef CONFIG_MEMORY_FAILURE
  4617. bool is_free_buddy_page(struct page *page)
  4618. {
  4619. struct zone *zone = page_zone(page);
  4620. unsigned long pfn = page_to_pfn(page);
  4621. unsigned long flags;
  4622. int order;
  4623. spin_lock_irqsave(&zone->lock, flags);
  4624. for (order = 0; order < MAX_ORDER; order++) {
  4625. struct page *page_head = page - (pfn & ((1 << order) - 1));
  4626. if (PageBuddy(page_head) && page_order(page_head) >= order)
  4627. break;
  4628. }
  4629. spin_unlock_irqrestore(&zone->lock, flags);
  4630. return order < MAX_ORDER;
  4631. }
  4632. #endif
  4633. static struct trace_print_flags pageflag_names[] = {
  4634. {1UL << PG_locked, "locked" },
  4635. {1UL << PG_error, "error" },
  4636. {1UL << PG_referenced, "referenced" },
  4637. {1UL << PG_uptodate, "uptodate" },
  4638. {1UL << PG_dirty, "dirty" },
  4639. {1UL << PG_lru, "lru" },
  4640. {1UL << PG_active, "active" },
  4641. {1UL << PG_slab, "slab" },
  4642. {1UL << PG_owner_priv_1, "owner_priv_1" },
  4643. {1UL << PG_arch_1, "arch_1" },
  4644. {1UL << PG_reserved, "reserved" },
  4645. {1UL << PG_private, "private" },
  4646. {1UL << PG_private_2, "private_2" },
  4647. {1UL << PG_writeback, "writeback" },
  4648. #ifdef CONFIG_PAGEFLAGS_EXTENDED
  4649. {1UL << PG_head, "head" },
  4650. {1UL << PG_tail, "tail" },
  4651. #else
  4652. {1UL << PG_compound, "compound" },
  4653. #endif
  4654. {1UL << PG_swapcache, "swapcache" },
  4655. {1UL << PG_mappedtodisk, "mappedtodisk" },
  4656. {1UL << PG_reclaim, "reclaim" },
  4657. {1UL << PG_buddy, "buddy" },
  4658. {1UL << PG_swapbacked, "swapbacked" },
  4659. {1UL << PG_unevictable, "unevictable" },
  4660. #ifdef CONFIG_MMU
  4661. {1UL << PG_mlocked, "mlocked" },
  4662. #endif
  4663. #ifdef CONFIG_ARCH_USES_PG_UNCACHED
  4664. {1UL << PG_uncached, "uncached" },
  4665. #endif
  4666. #ifdef CONFIG_MEMORY_FAILURE
  4667. {1UL << PG_hwpoison, "hwpoison" },
  4668. #endif
  4669. {-1UL, NULL },
  4670. };
  4671. static void dump_page_flags(unsigned long flags)
  4672. {
  4673. const char *delim = "";
  4674. unsigned long mask;
  4675. int i;
  4676. printk(KERN_ALERT "page flags: %#lx(", flags);
  4677. /* remove zone id */
  4678. flags &= (1UL << NR_PAGEFLAGS) - 1;
  4679. for (i = 0; pageflag_names[i].name && flags; i++) {
  4680. mask = pageflag_names[i].mask;
  4681. if ((flags & mask) != mask)
  4682. continue;
  4683. flags &= ~mask;
  4684. printk("%s%s", delim, pageflag_names[i].name);
  4685. delim = "|";
  4686. }
  4687. /* check for left over flags */
  4688. if (flags)
  4689. printk("%s%#lx", delim, flags);
  4690. printk(")\n");
  4691. }
  4692. void dump_page(struct page *page)
  4693. {
  4694. printk(KERN_ALERT
  4695. "page:%p count:%d mapcount:%d mapping:%p index:%#lx\n",
  4696. page, page_count(page), page_mapcount(page),
  4697. page->mapping, page->index);
  4698. dump_page_flags(page->flags);
  4699. }