memcontrol.c 132 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105
  1. /* memcontrol.c - Memory Controller
  2. *
  3. * Copyright IBM Corporation, 2007
  4. * Author Balbir Singh <balbir@linux.vnet.ibm.com>
  5. *
  6. * Copyright 2007 OpenVZ SWsoft Inc
  7. * Author: Pavel Emelianov <xemul@openvz.org>
  8. *
  9. * Memory thresholds
  10. * Copyright (C) 2009 Nokia Corporation
  11. * Author: Kirill A. Shutemov
  12. *
  13. * This program is free software; you can redistribute it and/or modify
  14. * it under the terms of the GNU General Public License as published by
  15. * the Free Software Foundation; either version 2 of the License, or
  16. * (at your option) any later version.
  17. *
  18. * This program is distributed in the hope that it will be useful,
  19. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  20. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  21. * GNU General Public License for more details.
  22. */
  23. #include <linux/res_counter.h>
  24. #include <linux/memcontrol.h>
  25. #include <linux/cgroup.h>
  26. #include <linux/mm.h>
  27. #include <linux/hugetlb.h>
  28. #include <linux/pagemap.h>
  29. #include <linux/smp.h>
  30. #include <linux/page-flags.h>
  31. #include <linux/backing-dev.h>
  32. #include <linux/bit_spinlock.h>
  33. #include <linux/rcupdate.h>
  34. #include <linux/limits.h>
  35. #include <linux/mutex.h>
  36. #include <linux/rbtree.h>
  37. #include <linux/slab.h>
  38. #include <linux/swap.h>
  39. #include <linux/swapops.h>
  40. #include <linux/spinlock.h>
  41. #include <linux/eventfd.h>
  42. #include <linux/sort.h>
  43. #include <linux/fs.h>
  44. #include <linux/seq_file.h>
  45. #include <linux/vmalloc.h>
  46. #include <linux/mm_inline.h>
  47. #include <linux/page_cgroup.h>
  48. #include <linux/cpu.h>
  49. #include <linux/oom.h>
  50. #include "internal.h"
  51. #include <asm/uaccess.h>
  52. #include <trace/events/vmscan.h>
  53. struct cgroup_subsys mem_cgroup_subsys __read_mostly;
  54. #define MEM_CGROUP_RECLAIM_RETRIES 5
  55. struct mem_cgroup *root_mem_cgroup __read_mostly;
  56. #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
  57. /* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */
  58. int do_swap_account __read_mostly;
  59. /* for remember boot option*/
  60. #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP_ENABLED
  61. static int really_do_swap_account __initdata = 1;
  62. #else
  63. static int really_do_swap_account __initdata = 0;
  64. #endif
  65. #else
  66. #define do_swap_account (0)
  67. #endif
  68. /*
  69. * Per memcg event counter is incremented at every pagein/pageout. This counter
  70. * is used for trigger some periodic events. This is straightforward and better
  71. * than using jiffies etc. to handle periodic memcg event.
  72. *
  73. * These values will be used as !((event) & ((1 <<(thresh)) - 1))
  74. */
  75. #define THRESHOLDS_EVENTS_THRESH (7) /* once in 128 */
  76. #define SOFTLIMIT_EVENTS_THRESH (10) /* once in 1024 */
  77. /*
  78. * Statistics for memory cgroup.
  79. */
  80. enum mem_cgroup_stat_index {
  81. /*
  82. * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
  83. */
  84. MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */
  85. MEM_CGROUP_STAT_RSS, /* # of pages charged as anon rss */
  86. MEM_CGROUP_STAT_FILE_MAPPED, /* # of pages charged as file rss */
  87. MEM_CGROUP_STAT_PGPGIN_COUNT, /* # of pages paged in */
  88. MEM_CGROUP_STAT_PGPGOUT_COUNT, /* # of pages paged out */
  89. MEM_CGROUP_STAT_SWAPOUT, /* # of pages, swapped out */
  90. MEM_CGROUP_STAT_DATA, /* end of data requires synchronization */
  91. /* incremented at every pagein/pageout */
  92. MEM_CGROUP_EVENTS = MEM_CGROUP_STAT_DATA,
  93. MEM_CGROUP_ON_MOVE, /* someone is moving account between groups */
  94. MEM_CGROUP_STAT_NSTATS,
  95. };
  96. struct mem_cgroup_stat_cpu {
  97. s64 count[MEM_CGROUP_STAT_NSTATS];
  98. };
  99. /*
  100. * per-zone information in memory controller.
  101. */
  102. struct mem_cgroup_per_zone {
  103. /*
  104. * spin_lock to protect the per cgroup LRU
  105. */
  106. struct list_head lists[NR_LRU_LISTS];
  107. unsigned long count[NR_LRU_LISTS];
  108. struct zone_reclaim_stat reclaim_stat;
  109. struct rb_node tree_node; /* RB tree node */
  110. unsigned long long usage_in_excess;/* Set to the value by which */
  111. /* the soft limit is exceeded*/
  112. bool on_tree;
  113. struct mem_cgroup *mem; /* Back pointer, we cannot */
  114. /* use container_of */
  115. };
  116. /* Macro for accessing counter */
  117. #define MEM_CGROUP_ZSTAT(mz, idx) ((mz)->count[(idx)])
  118. struct mem_cgroup_per_node {
  119. struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
  120. };
  121. struct mem_cgroup_lru_info {
  122. struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES];
  123. };
  124. /*
  125. * Cgroups above their limits are maintained in a RB-Tree, independent of
  126. * their hierarchy representation
  127. */
  128. struct mem_cgroup_tree_per_zone {
  129. struct rb_root rb_root;
  130. spinlock_t lock;
  131. };
  132. struct mem_cgroup_tree_per_node {
  133. struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
  134. };
  135. struct mem_cgroup_tree {
  136. struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
  137. };
  138. static struct mem_cgroup_tree soft_limit_tree __read_mostly;
  139. struct mem_cgroup_threshold {
  140. struct eventfd_ctx *eventfd;
  141. u64 threshold;
  142. };
  143. /* For threshold */
  144. struct mem_cgroup_threshold_ary {
  145. /* An array index points to threshold just below usage. */
  146. int current_threshold;
  147. /* Size of entries[] */
  148. unsigned int size;
  149. /* Array of thresholds */
  150. struct mem_cgroup_threshold entries[0];
  151. };
  152. struct mem_cgroup_thresholds {
  153. /* Primary thresholds array */
  154. struct mem_cgroup_threshold_ary *primary;
  155. /*
  156. * Spare threshold array.
  157. * This is needed to make mem_cgroup_unregister_event() "never fail".
  158. * It must be able to store at least primary->size - 1 entries.
  159. */
  160. struct mem_cgroup_threshold_ary *spare;
  161. };
  162. /* for OOM */
  163. struct mem_cgroup_eventfd_list {
  164. struct list_head list;
  165. struct eventfd_ctx *eventfd;
  166. };
  167. static void mem_cgroup_threshold(struct mem_cgroup *mem);
  168. static void mem_cgroup_oom_notify(struct mem_cgroup *mem);
  169. /*
  170. * The memory controller data structure. The memory controller controls both
  171. * page cache and RSS per cgroup. We would eventually like to provide
  172. * statistics based on the statistics developed by Rik Van Riel for clock-pro,
  173. * to help the administrator determine what knobs to tune.
  174. *
  175. * TODO: Add a water mark for the memory controller. Reclaim will begin when
  176. * we hit the water mark. May be even add a low water mark, such that
  177. * no reclaim occurs from a cgroup at it's low water mark, this is
  178. * a feature that will be implemented much later in the future.
  179. */
  180. struct mem_cgroup {
  181. struct cgroup_subsys_state css;
  182. /*
  183. * the counter to account for memory usage
  184. */
  185. struct res_counter res;
  186. /*
  187. * the counter to account for mem+swap usage.
  188. */
  189. struct res_counter memsw;
  190. /*
  191. * Per cgroup active and inactive list, similar to the
  192. * per zone LRU lists.
  193. */
  194. struct mem_cgroup_lru_info info;
  195. /*
  196. protect against reclaim related member.
  197. */
  198. spinlock_t reclaim_param_lock;
  199. /*
  200. * While reclaiming in a hierarchy, we cache the last child we
  201. * reclaimed from.
  202. */
  203. int last_scanned_child;
  204. /*
  205. * Should the accounting and control be hierarchical, per subtree?
  206. */
  207. bool use_hierarchy;
  208. atomic_t oom_lock;
  209. atomic_t refcnt;
  210. unsigned int swappiness;
  211. /* OOM-Killer disable */
  212. int oom_kill_disable;
  213. /* set when res.limit == memsw.limit */
  214. bool memsw_is_minimum;
  215. /* protect arrays of thresholds */
  216. struct mutex thresholds_lock;
  217. /* thresholds for memory usage. RCU-protected */
  218. struct mem_cgroup_thresholds thresholds;
  219. /* thresholds for mem+swap usage. RCU-protected */
  220. struct mem_cgroup_thresholds memsw_thresholds;
  221. /* For oom notifier event fd */
  222. struct list_head oom_notify;
  223. /*
  224. * Should we move charges of a task when a task is moved into this
  225. * mem_cgroup ? And what type of charges should we move ?
  226. */
  227. unsigned long move_charge_at_immigrate;
  228. /*
  229. * percpu counter.
  230. */
  231. struct mem_cgroup_stat_cpu *stat;
  232. /*
  233. * used when a cpu is offlined or other synchronizations
  234. * See mem_cgroup_read_stat().
  235. */
  236. struct mem_cgroup_stat_cpu nocpu_base;
  237. spinlock_t pcp_counter_lock;
  238. };
  239. /* Stuffs for move charges at task migration. */
  240. /*
  241. * Types of charges to be moved. "move_charge_at_immitgrate" is treated as a
  242. * left-shifted bitmap of these types.
  243. */
  244. enum move_type {
  245. MOVE_CHARGE_TYPE_ANON, /* private anonymous page and swap of it */
  246. MOVE_CHARGE_TYPE_FILE, /* file page(including tmpfs) and swap of it */
  247. NR_MOVE_TYPE,
  248. };
  249. /* "mc" and its members are protected by cgroup_mutex */
  250. static struct move_charge_struct {
  251. spinlock_t lock; /* for from, to */
  252. struct mem_cgroup *from;
  253. struct mem_cgroup *to;
  254. unsigned long precharge;
  255. unsigned long moved_charge;
  256. unsigned long moved_swap;
  257. struct task_struct *moving_task; /* a task moving charges */
  258. wait_queue_head_t waitq; /* a waitq for other context */
  259. } mc = {
  260. .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
  261. .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
  262. };
  263. static bool move_anon(void)
  264. {
  265. return test_bit(MOVE_CHARGE_TYPE_ANON,
  266. &mc.to->move_charge_at_immigrate);
  267. }
  268. static bool move_file(void)
  269. {
  270. return test_bit(MOVE_CHARGE_TYPE_FILE,
  271. &mc.to->move_charge_at_immigrate);
  272. }
  273. /*
  274. * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
  275. * limit reclaim to prevent infinite loops, if they ever occur.
  276. */
  277. #define MEM_CGROUP_MAX_RECLAIM_LOOPS (100)
  278. #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS (2)
  279. enum charge_type {
  280. MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
  281. MEM_CGROUP_CHARGE_TYPE_MAPPED,
  282. MEM_CGROUP_CHARGE_TYPE_SHMEM, /* used by page migration of shmem */
  283. MEM_CGROUP_CHARGE_TYPE_FORCE, /* used by force_empty */
  284. MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
  285. MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */
  286. NR_CHARGE_TYPE,
  287. };
  288. /* only for here (for easy reading.) */
  289. #define PCGF_CACHE (1UL << PCG_CACHE)
  290. #define PCGF_USED (1UL << PCG_USED)
  291. #define PCGF_LOCK (1UL << PCG_LOCK)
  292. /* Not used, but added here for completeness */
  293. #define PCGF_ACCT (1UL << PCG_ACCT)
  294. /* for encoding cft->private value on file */
  295. #define _MEM (0)
  296. #define _MEMSWAP (1)
  297. #define _OOM_TYPE (2)
  298. #define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val))
  299. #define MEMFILE_TYPE(val) (((val) >> 16) & 0xffff)
  300. #define MEMFILE_ATTR(val) ((val) & 0xffff)
  301. /* Used for OOM nofiier */
  302. #define OOM_CONTROL (0)
  303. /*
  304. * Reclaim flags for mem_cgroup_hierarchical_reclaim
  305. */
  306. #define MEM_CGROUP_RECLAIM_NOSWAP_BIT 0x0
  307. #define MEM_CGROUP_RECLAIM_NOSWAP (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
  308. #define MEM_CGROUP_RECLAIM_SHRINK_BIT 0x1
  309. #define MEM_CGROUP_RECLAIM_SHRINK (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
  310. #define MEM_CGROUP_RECLAIM_SOFT_BIT 0x2
  311. #define MEM_CGROUP_RECLAIM_SOFT (1 << MEM_CGROUP_RECLAIM_SOFT_BIT)
  312. static void mem_cgroup_get(struct mem_cgroup *mem);
  313. static void mem_cgroup_put(struct mem_cgroup *mem);
  314. static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem);
  315. static void drain_all_stock_async(void);
  316. static struct mem_cgroup_per_zone *
  317. mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid)
  318. {
  319. return &mem->info.nodeinfo[nid]->zoneinfo[zid];
  320. }
  321. struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *mem)
  322. {
  323. return &mem->css;
  324. }
  325. static struct mem_cgroup_per_zone *
  326. page_cgroup_zoneinfo(struct page_cgroup *pc)
  327. {
  328. struct mem_cgroup *mem = pc->mem_cgroup;
  329. int nid = page_cgroup_nid(pc);
  330. int zid = page_cgroup_zid(pc);
  331. if (!mem)
  332. return NULL;
  333. return mem_cgroup_zoneinfo(mem, nid, zid);
  334. }
  335. static struct mem_cgroup_tree_per_zone *
  336. soft_limit_tree_node_zone(int nid, int zid)
  337. {
  338. return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
  339. }
  340. static struct mem_cgroup_tree_per_zone *
  341. soft_limit_tree_from_page(struct page *page)
  342. {
  343. int nid = page_to_nid(page);
  344. int zid = page_zonenum(page);
  345. return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
  346. }
  347. static void
  348. __mem_cgroup_insert_exceeded(struct mem_cgroup *mem,
  349. struct mem_cgroup_per_zone *mz,
  350. struct mem_cgroup_tree_per_zone *mctz,
  351. unsigned long long new_usage_in_excess)
  352. {
  353. struct rb_node **p = &mctz->rb_root.rb_node;
  354. struct rb_node *parent = NULL;
  355. struct mem_cgroup_per_zone *mz_node;
  356. if (mz->on_tree)
  357. return;
  358. mz->usage_in_excess = new_usage_in_excess;
  359. if (!mz->usage_in_excess)
  360. return;
  361. while (*p) {
  362. parent = *p;
  363. mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
  364. tree_node);
  365. if (mz->usage_in_excess < mz_node->usage_in_excess)
  366. p = &(*p)->rb_left;
  367. /*
  368. * We can't avoid mem cgroups that are over their soft
  369. * limit by the same amount
  370. */
  371. else if (mz->usage_in_excess >= mz_node->usage_in_excess)
  372. p = &(*p)->rb_right;
  373. }
  374. rb_link_node(&mz->tree_node, parent, p);
  375. rb_insert_color(&mz->tree_node, &mctz->rb_root);
  376. mz->on_tree = true;
  377. }
  378. static void
  379. __mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
  380. struct mem_cgroup_per_zone *mz,
  381. struct mem_cgroup_tree_per_zone *mctz)
  382. {
  383. if (!mz->on_tree)
  384. return;
  385. rb_erase(&mz->tree_node, &mctz->rb_root);
  386. mz->on_tree = false;
  387. }
  388. static void
  389. mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
  390. struct mem_cgroup_per_zone *mz,
  391. struct mem_cgroup_tree_per_zone *mctz)
  392. {
  393. spin_lock(&mctz->lock);
  394. __mem_cgroup_remove_exceeded(mem, mz, mctz);
  395. spin_unlock(&mctz->lock);
  396. }
  397. static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page)
  398. {
  399. unsigned long long excess;
  400. struct mem_cgroup_per_zone *mz;
  401. struct mem_cgroup_tree_per_zone *mctz;
  402. int nid = page_to_nid(page);
  403. int zid = page_zonenum(page);
  404. mctz = soft_limit_tree_from_page(page);
  405. /*
  406. * Necessary to update all ancestors when hierarchy is used.
  407. * because their event counter is not touched.
  408. */
  409. for (; mem; mem = parent_mem_cgroup(mem)) {
  410. mz = mem_cgroup_zoneinfo(mem, nid, zid);
  411. excess = res_counter_soft_limit_excess(&mem->res);
  412. /*
  413. * We have to update the tree if mz is on RB-tree or
  414. * mem is over its softlimit.
  415. */
  416. if (excess || mz->on_tree) {
  417. spin_lock(&mctz->lock);
  418. /* if on-tree, remove it */
  419. if (mz->on_tree)
  420. __mem_cgroup_remove_exceeded(mem, mz, mctz);
  421. /*
  422. * Insert again. mz->usage_in_excess will be updated.
  423. * If excess is 0, no tree ops.
  424. */
  425. __mem_cgroup_insert_exceeded(mem, mz, mctz, excess);
  426. spin_unlock(&mctz->lock);
  427. }
  428. }
  429. }
  430. static void mem_cgroup_remove_from_trees(struct mem_cgroup *mem)
  431. {
  432. int node, zone;
  433. struct mem_cgroup_per_zone *mz;
  434. struct mem_cgroup_tree_per_zone *mctz;
  435. for_each_node_state(node, N_POSSIBLE) {
  436. for (zone = 0; zone < MAX_NR_ZONES; zone++) {
  437. mz = mem_cgroup_zoneinfo(mem, node, zone);
  438. mctz = soft_limit_tree_node_zone(node, zone);
  439. mem_cgroup_remove_exceeded(mem, mz, mctz);
  440. }
  441. }
  442. }
  443. static inline unsigned long mem_cgroup_get_excess(struct mem_cgroup *mem)
  444. {
  445. return res_counter_soft_limit_excess(&mem->res) >> PAGE_SHIFT;
  446. }
  447. static struct mem_cgroup_per_zone *
  448. __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
  449. {
  450. struct rb_node *rightmost = NULL;
  451. struct mem_cgroup_per_zone *mz;
  452. retry:
  453. mz = NULL;
  454. rightmost = rb_last(&mctz->rb_root);
  455. if (!rightmost)
  456. goto done; /* Nothing to reclaim from */
  457. mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
  458. /*
  459. * Remove the node now but someone else can add it back,
  460. * we will to add it back at the end of reclaim to its correct
  461. * position in the tree.
  462. */
  463. __mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
  464. if (!res_counter_soft_limit_excess(&mz->mem->res) ||
  465. !css_tryget(&mz->mem->css))
  466. goto retry;
  467. done:
  468. return mz;
  469. }
  470. static struct mem_cgroup_per_zone *
  471. mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
  472. {
  473. struct mem_cgroup_per_zone *mz;
  474. spin_lock(&mctz->lock);
  475. mz = __mem_cgroup_largest_soft_limit_node(mctz);
  476. spin_unlock(&mctz->lock);
  477. return mz;
  478. }
  479. /*
  480. * Implementation Note: reading percpu statistics for memcg.
  481. *
  482. * Both of vmstat[] and percpu_counter has threshold and do periodic
  483. * synchronization to implement "quick" read. There are trade-off between
  484. * reading cost and precision of value. Then, we may have a chance to implement
  485. * a periodic synchronizion of counter in memcg's counter.
  486. *
  487. * But this _read() function is used for user interface now. The user accounts
  488. * memory usage by memory cgroup and he _always_ requires exact value because
  489. * he accounts memory. Even if we provide quick-and-fuzzy read, we always
  490. * have to visit all online cpus and make sum. So, for now, unnecessary
  491. * synchronization is not implemented. (just implemented for cpu hotplug)
  492. *
  493. * If there are kernel internal actions which can make use of some not-exact
  494. * value, and reading all cpu value can be performance bottleneck in some
  495. * common workload, threashold and synchonization as vmstat[] should be
  496. * implemented.
  497. */
  498. static s64 mem_cgroup_read_stat(struct mem_cgroup *mem,
  499. enum mem_cgroup_stat_index idx)
  500. {
  501. int cpu;
  502. s64 val = 0;
  503. get_online_cpus();
  504. for_each_online_cpu(cpu)
  505. val += per_cpu(mem->stat->count[idx], cpu);
  506. #ifdef CONFIG_HOTPLUG_CPU
  507. spin_lock(&mem->pcp_counter_lock);
  508. val += mem->nocpu_base.count[idx];
  509. spin_unlock(&mem->pcp_counter_lock);
  510. #endif
  511. put_online_cpus();
  512. return val;
  513. }
  514. static s64 mem_cgroup_local_usage(struct mem_cgroup *mem)
  515. {
  516. s64 ret;
  517. ret = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS);
  518. ret += mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE);
  519. return ret;
  520. }
  521. static void mem_cgroup_swap_statistics(struct mem_cgroup *mem,
  522. bool charge)
  523. {
  524. int val = (charge) ? 1 : -1;
  525. this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_SWAPOUT], val);
  526. }
  527. static void mem_cgroup_charge_statistics(struct mem_cgroup *mem,
  528. bool file, int nr_pages)
  529. {
  530. preempt_disable();
  531. if (file)
  532. __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_CACHE], nr_pages);
  533. else
  534. __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_RSS], nr_pages);
  535. /* pagein of a big page is an event. So, ignore page size */
  536. if (nr_pages > 0)
  537. __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGIN_COUNT]);
  538. else {
  539. __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGOUT_COUNT]);
  540. nr_pages = -nr_pages; /* for event */
  541. }
  542. __this_cpu_add(mem->stat->count[MEM_CGROUP_EVENTS], nr_pages);
  543. preempt_enable();
  544. }
  545. static unsigned long mem_cgroup_get_local_zonestat(struct mem_cgroup *mem,
  546. enum lru_list idx)
  547. {
  548. int nid, zid;
  549. struct mem_cgroup_per_zone *mz;
  550. u64 total = 0;
  551. for_each_online_node(nid)
  552. for (zid = 0; zid < MAX_NR_ZONES; zid++) {
  553. mz = mem_cgroup_zoneinfo(mem, nid, zid);
  554. total += MEM_CGROUP_ZSTAT(mz, idx);
  555. }
  556. return total;
  557. }
  558. static bool __memcg_event_check(struct mem_cgroup *mem, int event_mask_shift)
  559. {
  560. s64 val;
  561. val = this_cpu_read(mem->stat->count[MEM_CGROUP_EVENTS]);
  562. return !(val & ((1 << event_mask_shift) - 1));
  563. }
  564. /*
  565. * Check events in order.
  566. *
  567. */
  568. static void memcg_check_events(struct mem_cgroup *mem, struct page *page)
  569. {
  570. /* threshold event is triggered in finer grain than soft limit */
  571. if (unlikely(__memcg_event_check(mem, THRESHOLDS_EVENTS_THRESH))) {
  572. mem_cgroup_threshold(mem);
  573. if (unlikely(__memcg_event_check(mem, SOFTLIMIT_EVENTS_THRESH)))
  574. mem_cgroup_update_tree(mem, page);
  575. }
  576. }
  577. static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
  578. {
  579. return container_of(cgroup_subsys_state(cont,
  580. mem_cgroup_subsys_id), struct mem_cgroup,
  581. css);
  582. }
  583. struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
  584. {
  585. /*
  586. * mm_update_next_owner() may clear mm->owner to NULL
  587. * if it races with swapoff, page migration, etc.
  588. * So this can be called with p == NULL.
  589. */
  590. if (unlikely(!p))
  591. return NULL;
  592. return container_of(task_subsys_state(p, mem_cgroup_subsys_id),
  593. struct mem_cgroup, css);
  594. }
  595. static struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
  596. {
  597. struct mem_cgroup *mem = NULL;
  598. if (!mm)
  599. return NULL;
  600. /*
  601. * Because we have no locks, mm->owner's may be being moved to other
  602. * cgroup. We use css_tryget() here even if this looks
  603. * pessimistic (rather than adding locks here).
  604. */
  605. rcu_read_lock();
  606. do {
  607. mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
  608. if (unlikely(!mem))
  609. break;
  610. } while (!css_tryget(&mem->css));
  611. rcu_read_unlock();
  612. return mem;
  613. }
  614. /* The caller has to guarantee "mem" exists before calling this */
  615. static struct mem_cgroup *mem_cgroup_start_loop(struct mem_cgroup *mem)
  616. {
  617. struct cgroup_subsys_state *css;
  618. int found;
  619. if (!mem) /* ROOT cgroup has the smallest ID */
  620. return root_mem_cgroup; /*css_put/get against root is ignored*/
  621. if (!mem->use_hierarchy) {
  622. if (css_tryget(&mem->css))
  623. return mem;
  624. return NULL;
  625. }
  626. rcu_read_lock();
  627. /*
  628. * searching a memory cgroup which has the smallest ID under given
  629. * ROOT cgroup. (ID >= 1)
  630. */
  631. css = css_get_next(&mem_cgroup_subsys, 1, &mem->css, &found);
  632. if (css && css_tryget(css))
  633. mem = container_of(css, struct mem_cgroup, css);
  634. else
  635. mem = NULL;
  636. rcu_read_unlock();
  637. return mem;
  638. }
  639. static struct mem_cgroup *mem_cgroup_get_next(struct mem_cgroup *iter,
  640. struct mem_cgroup *root,
  641. bool cond)
  642. {
  643. int nextid = css_id(&iter->css) + 1;
  644. int found;
  645. int hierarchy_used;
  646. struct cgroup_subsys_state *css;
  647. hierarchy_used = iter->use_hierarchy;
  648. css_put(&iter->css);
  649. /* If no ROOT, walk all, ignore hierarchy */
  650. if (!cond || (root && !hierarchy_used))
  651. return NULL;
  652. if (!root)
  653. root = root_mem_cgroup;
  654. do {
  655. iter = NULL;
  656. rcu_read_lock();
  657. css = css_get_next(&mem_cgroup_subsys, nextid,
  658. &root->css, &found);
  659. if (css && css_tryget(css))
  660. iter = container_of(css, struct mem_cgroup, css);
  661. rcu_read_unlock();
  662. /* If css is NULL, no more cgroups will be found */
  663. nextid = found + 1;
  664. } while (css && !iter);
  665. return iter;
  666. }
  667. /*
  668. * for_eacn_mem_cgroup_tree() for visiting all cgroup under tree. Please
  669. * be careful that "break" loop is not allowed. We have reference count.
  670. * Instead of that modify "cond" to be false and "continue" to exit the loop.
  671. */
  672. #define for_each_mem_cgroup_tree_cond(iter, root, cond) \
  673. for (iter = mem_cgroup_start_loop(root);\
  674. iter != NULL;\
  675. iter = mem_cgroup_get_next(iter, root, cond))
  676. #define for_each_mem_cgroup_tree(iter, root) \
  677. for_each_mem_cgroup_tree_cond(iter, root, true)
  678. #define for_each_mem_cgroup_all(iter) \
  679. for_each_mem_cgroup_tree_cond(iter, NULL, true)
  680. static inline bool mem_cgroup_is_root(struct mem_cgroup *mem)
  681. {
  682. return (mem == root_mem_cgroup);
  683. }
  684. /*
  685. * Following LRU functions are allowed to be used without PCG_LOCK.
  686. * Operations are called by routine of global LRU independently from memcg.
  687. * What we have to take care of here is validness of pc->mem_cgroup.
  688. *
  689. * Changes to pc->mem_cgroup happens when
  690. * 1. charge
  691. * 2. moving account
  692. * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
  693. * It is added to LRU before charge.
  694. * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
  695. * When moving account, the page is not on LRU. It's isolated.
  696. */
  697. void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru)
  698. {
  699. struct page_cgroup *pc;
  700. struct mem_cgroup_per_zone *mz;
  701. if (mem_cgroup_disabled())
  702. return;
  703. pc = lookup_page_cgroup(page);
  704. /* can happen while we handle swapcache. */
  705. if (!TestClearPageCgroupAcctLRU(pc))
  706. return;
  707. VM_BUG_ON(!pc->mem_cgroup);
  708. /*
  709. * We don't check PCG_USED bit. It's cleared when the "page" is finally
  710. * removed from global LRU.
  711. */
  712. mz = page_cgroup_zoneinfo(pc);
  713. /* huge page split is done under lru_lock. so, we have no races. */
  714. MEM_CGROUP_ZSTAT(mz, lru) -= 1 << compound_order(page);
  715. if (mem_cgroup_is_root(pc->mem_cgroup))
  716. return;
  717. VM_BUG_ON(list_empty(&pc->lru));
  718. list_del_init(&pc->lru);
  719. }
  720. void mem_cgroup_del_lru(struct page *page)
  721. {
  722. mem_cgroup_del_lru_list(page, page_lru(page));
  723. }
  724. /*
  725. * Writeback is about to end against a page which has been marked for immediate
  726. * reclaim. If it still appears to be reclaimable, move it to the tail of the
  727. * inactive list.
  728. */
  729. void mem_cgroup_rotate_reclaimable_page(struct page *page)
  730. {
  731. struct mem_cgroup_per_zone *mz;
  732. struct page_cgroup *pc;
  733. enum lru_list lru = page_lru(page);
  734. if (mem_cgroup_disabled())
  735. return;
  736. pc = lookup_page_cgroup(page);
  737. /* unused or root page is not rotated. */
  738. if (!PageCgroupUsed(pc))
  739. return;
  740. /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */
  741. smp_rmb();
  742. if (mem_cgroup_is_root(pc->mem_cgroup))
  743. return;
  744. mz = page_cgroup_zoneinfo(pc);
  745. list_move_tail(&pc->lru, &mz->lists[lru]);
  746. }
  747. void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru)
  748. {
  749. struct mem_cgroup_per_zone *mz;
  750. struct page_cgroup *pc;
  751. if (mem_cgroup_disabled())
  752. return;
  753. pc = lookup_page_cgroup(page);
  754. /* unused or root page is not rotated. */
  755. if (!PageCgroupUsed(pc))
  756. return;
  757. /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */
  758. smp_rmb();
  759. if (mem_cgroup_is_root(pc->mem_cgroup))
  760. return;
  761. mz = page_cgroup_zoneinfo(pc);
  762. list_move(&pc->lru, &mz->lists[lru]);
  763. }
  764. void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru)
  765. {
  766. struct page_cgroup *pc;
  767. struct mem_cgroup_per_zone *mz;
  768. if (mem_cgroup_disabled())
  769. return;
  770. pc = lookup_page_cgroup(page);
  771. VM_BUG_ON(PageCgroupAcctLRU(pc));
  772. if (!PageCgroupUsed(pc))
  773. return;
  774. /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */
  775. smp_rmb();
  776. mz = page_cgroup_zoneinfo(pc);
  777. /* huge page split is done under lru_lock. so, we have no races. */
  778. MEM_CGROUP_ZSTAT(mz, lru) += 1 << compound_order(page);
  779. SetPageCgroupAcctLRU(pc);
  780. if (mem_cgroup_is_root(pc->mem_cgroup))
  781. return;
  782. list_add(&pc->lru, &mz->lists[lru]);
  783. }
  784. /*
  785. * At handling SwapCache, pc->mem_cgroup may be changed while it's linked to
  786. * lru because the page may.be reused after it's fully uncharged (because of
  787. * SwapCache behavior).To handle that, unlink page_cgroup from LRU when charge
  788. * it again. This function is only used to charge SwapCache. It's done under
  789. * lock_page and expected that zone->lru_lock is never held.
  790. */
  791. static void mem_cgroup_lru_del_before_commit_swapcache(struct page *page)
  792. {
  793. unsigned long flags;
  794. struct zone *zone = page_zone(page);
  795. struct page_cgroup *pc = lookup_page_cgroup(page);
  796. spin_lock_irqsave(&zone->lru_lock, flags);
  797. /*
  798. * Forget old LRU when this page_cgroup is *not* used. This Used bit
  799. * is guarded by lock_page() because the page is SwapCache.
  800. */
  801. if (!PageCgroupUsed(pc))
  802. mem_cgroup_del_lru_list(page, page_lru(page));
  803. spin_unlock_irqrestore(&zone->lru_lock, flags);
  804. }
  805. static void mem_cgroup_lru_add_after_commit_swapcache(struct page *page)
  806. {
  807. unsigned long flags;
  808. struct zone *zone = page_zone(page);
  809. struct page_cgroup *pc = lookup_page_cgroup(page);
  810. spin_lock_irqsave(&zone->lru_lock, flags);
  811. /* link when the page is linked to LRU but page_cgroup isn't */
  812. if (PageLRU(page) && !PageCgroupAcctLRU(pc))
  813. mem_cgroup_add_lru_list(page, page_lru(page));
  814. spin_unlock_irqrestore(&zone->lru_lock, flags);
  815. }
  816. void mem_cgroup_move_lists(struct page *page,
  817. enum lru_list from, enum lru_list to)
  818. {
  819. if (mem_cgroup_disabled())
  820. return;
  821. mem_cgroup_del_lru_list(page, from);
  822. mem_cgroup_add_lru_list(page, to);
  823. }
  824. int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
  825. {
  826. int ret;
  827. struct mem_cgroup *curr = NULL;
  828. struct task_struct *p;
  829. p = find_lock_task_mm(task);
  830. if (!p)
  831. return 0;
  832. curr = try_get_mem_cgroup_from_mm(p->mm);
  833. task_unlock(p);
  834. if (!curr)
  835. return 0;
  836. /*
  837. * We should check use_hierarchy of "mem" not "curr". Because checking
  838. * use_hierarchy of "curr" here make this function true if hierarchy is
  839. * enabled in "curr" and "curr" is a child of "mem" in *cgroup*
  840. * hierarchy(even if use_hierarchy is disabled in "mem").
  841. */
  842. if (mem->use_hierarchy)
  843. ret = css_is_ancestor(&curr->css, &mem->css);
  844. else
  845. ret = (curr == mem);
  846. css_put(&curr->css);
  847. return ret;
  848. }
  849. static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_pages)
  850. {
  851. unsigned long active;
  852. unsigned long inactive;
  853. unsigned long gb;
  854. unsigned long inactive_ratio;
  855. inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_ANON);
  856. active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_ANON);
  857. gb = (inactive + active) >> (30 - PAGE_SHIFT);
  858. if (gb)
  859. inactive_ratio = int_sqrt(10 * gb);
  860. else
  861. inactive_ratio = 1;
  862. if (present_pages) {
  863. present_pages[0] = inactive;
  864. present_pages[1] = active;
  865. }
  866. return inactive_ratio;
  867. }
  868. int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg)
  869. {
  870. unsigned long active;
  871. unsigned long inactive;
  872. unsigned long present_pages[2];
  873. unsigned long inactive_ratio;
  874. inactive_ratio = calc_inactive_ratio(memcg, present_pages);
  875. inactive = present_pages[0];
  876. active = present_pages[1];
  877. if (inactive * inactive_ratio < active)
  878. return 1;
  879. return 0;
  880. }
  881. int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg)
  882. {
  883. unsigned long active;
  884. unsigned long inactive;
  885. inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_FILE);
  886. active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_FILE);
  887. return (active > inactive);
  888. }
  889. unsigned long mem_cgroup_zone_nr_pages(struct mem_cgroup *memcg,
  890. struct zone *zone,
  891. enum lru_list lru)
  892. {
  893. int nid = zone_to_nid(zone);
  894. int zid = zone_idx(zone);
  895. struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
  896. return MEM_CGROUP_ZSTAT(mz, lru);
  897. }
  898. struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg,
  899. struct zone *zone)
  900. {
  901. int nid = zone_to_nid(zone);
  902. int zid = zone_idx(zone);
  903. struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
  904. return &mz->reclaim_stat;
  905. }
  906. struct zone_reclaim_stat *
  907. mem_cgroup_get_reclaim_stat_from_page(struct page *page)
  908. {
  909. struct page_cgroup *pc;
  910. struct mem_cgroup_per_zone *mz;
  911. if (mem_cgroup_disabled())
  912. return NULL;
  913. pc = lookup_page_cgroup(page);
  914. if (!PageCgroupUsed(pc))
  915. return NULL;
  916. /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */
  917. smp_rmb();
  918. mz = page_cgroup_zoneinfo(pc);
  919. if (!mz)
  920. return NULL;
  921. return &mz->reclaim_stat;
  922. }
  923. unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
  924. struct list_head *dst,
  925. unsigned long *scanned, int order,
  926. int mode, struct zone *z,
  927. struct mem_cgroup *mem_cont,
  928. int active, int file)
  929. {
  930. unsigned long nr_taken = 0;
  931. struct page *page;
  932. unsigned long scan;
  933. LIST_HEAD(pc_list);
  934. struct list_head *src;
  935. struct page_cgroup *pc, *tmp;
  936. int nid = zone_to_nid(z);
  937. int zid = zone_idx(z);
  938. struct mem_cgroup_per_zone *mz;
  939. int lru = LRU_FILE * file + active;
  940. int ret;
  941. BUG_ON(!mem_cont);
  942. mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
  943. src = &mz->lists[lru];
  944. scan = 0;
  945. list_for_each_entry_safe_reverse(pc, tmp, src, lru) {
  946. if (scan >= nr_to_scan)
  947. break;
  948. page = pc->page;
  949. if (unlikely(!PageCgroupUsed(pc)))
  950. continue;
  951. if (unlikely(!PageLRU(page)))
  952. continue;
  953. scan++;
  954. ret = __isolate_lru_page(page, mode, file);
  955. switch (ret) {
  956. case 0:
  957. list_move(&page->lru, dst);
  958. mem_cgroup_del_lru(page);
  959. nr_taken += hpage_nr_pages(page);
  960. break;
  961. case -EBUSY:
  962. /* we don't affect global LRU but rotate in our LRU */
  963. mem_cgroup_rotate_lru_list(page, page_lru(page));
  964. break;
  965. default:
  966. break;
  967. }
  968. }
  969. *scanned = scan;
  970. trace_mm_vmscan_memcg_isolate(0, nr_to_scan, scan, nr_taken,
  971. 0, 0, 0, mode);
  972. return nr_taken;
  973. }
  974. #define mem_cgroup_from_res_counter(counter, member) \
  975. container_of(counter, struct mem_cgroup, member)
  976. static bool mem_cgroup_check_under_limit(struct mem_cgroup *mem)
  977. {
  978. if (do_swap_account) {
  979. if (res_counter_check_under_limit(&mem->res) &&
  980. res_counter_check_under_limit(&mem->memsw))
  981. return true;
  982. } else
  983. if (res_counter_check_under_limit(&mem->res))
  984. return true;
  985. return false;
  986. }
  987. /**
  988. * mem_cgroup_check_margin - check if the memory cgroup allows charging
  989. * @mem: memory cgroup to check
  990. * @bytes: the number of bytes the caller intends to charge
  991. *
  992. * Returns a boolean value on whether @mem can be charged @bytes or
  993. * whether this would exceed the limit.
  994. */
  995. static bool mem_cgroup_check_margin(struct mem_cgroup *mem, unsigned long bytes)
  996. {
  997. if (!res_counter_check_margin(&mem->res, bytes))
  998. return false;
  999. if (do_swap_account && !res_counter_check_margin(&mem->memsw, bytes))
  1000. return false;
  1001. return true;
  1002. }
  1003. static unsigned int get_swappiness(struct mem_cgroup *memcg)
  1004. {
  1005. struct cgroup *cgrp = memcg->css.cgroup;
  1006. unsigned int swappiness;
  1007. /* root ? */
  1008. if (cgrp->parent == NULL)
  1009. return vm_swappiness;
  1010. spin_lock(&memcg->reclaim_param_lock);
  1011. swappiness = memcg->swappiness;
  1012. spin_unlock(&memcg->reclaim_param_lock);
  1013. return swappiness;
  1014. }
  1015. static void mem_cgroup_start_move(struct mem_cgroup *mem)
  1016. {
  1017. int cpu;
  1018. get_online_cpus();
  1019. spin_lock(&mem->pcp_counter_lock);
  1020. for_each_online_cpu(cpu)
  1021. per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) += 1;
  1022. mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] += 1;
  1023. spin_unlock(&mem->pcp_counter_lock);
  1024. put_online_cpus();
  1025. synchronize_rcu();
  1026. }
  1027. static void mem_cgroup_end_move(struct mem_cgroup *mem)
  1028. {
  1029. int cpu;
  1030. if (!mem)
  1031. return;
  1032. get_online_cpus();
  1033. spin_lock(&mem->pcp_counter_lock);
  1034. for_each_online_cpu(cpu)
  1035. per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) -= 1;
  1036. mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] -= 1;
  1037. spin_unlock(&mem->pcp_counter_lock);
  1038. put_online_cpus();
  1039. }
  1040. /*
  1041. * 2 routines for checking "mem" is under move_account() or not.
  1042. *
  1043. * mem_cgroup_stealed() - checking a cgroup is mc.from or not. This is used
  1044. * for avoiding race in accounting. If true,
  1045. * pc->mem_cgroup may be overwritten.
  1046. *
  1047. * mem_cgroup_under_move() - checking a cgroup is mc.from or mc.to or
  1048. * under hierarchy of moving cgroups. This is for
  1049. * waiting at hith-memory prressure caused by "move".
  1050. */
  1051. static bool mem_cgroup_stealed(struct mem_cgroup *mem)
  1052. {
  1053. VM_BUG_ON(!rcu_read_lock_held());
  1054. return this_cpu_read(mem->stat->count[MEM_CGROUP_ON_MOVE]) > 0;
  1055. }
  1056. static bool mem_cgroup_under_move(struct mem_cgroup *mem)
  1057. {
  1058. struct mem_cgroup *from;
  1059. struct mem_cgroup *to;
  1060. bool ret = false;
  1061. /*
  1062. * Unlike task_move routines, we access mc.to, mc.from not under
  1063. * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
  1064. */
  1065. spin_lock(&mc.lock);
  1066. from = mc.from;
  1067. to = mc.to;
  1068. if (!from)
  1069. goto unlock;
  1070. if (from == mem || to == mem
  1071. || (mem->use_hierarchy && css_is_ancestor(&from->css, &mem->css))
  1072. || (mem->use_hierarchy && css_is_ancestor(&to->css, &mem->css)))
  1073. ret = true;
  1074. unlock:
  1075. spin_unlock(&mc.lock);
  1076. return ret;
  1077. }
  1078. static bool mem_cgroup_wait_acct_move(struct mem_cgroup *mem)
  1079. {
  1080. if (mc.moving_task && current != mc.moving_task) {
  1081. if (mem_cgroup_under_move(mem)) {
  1082. DEFINE_WAIT(wait);
  1083. prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
  1084. /* moving charge context might have finished. */
  1085. if (mc.moving_task)
  1086. schedule();
  1087. finish_wait(&mc.waitq, &wait);
  1088. return true;
  1089. }
  1090. }
  1091. return false;
  1092. }
  1093. /**
  1094. * mem_cgroup_print_oom_info: Called from OOM with tasklist_lock held in read mode.
  1095. * @memcg: The memory cgroup that went over limit
  1096. * @p: Task that is going to be killed
  1097. *
  1098. * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
  1099. * enabled
  1100. */
  1101. void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
  1102. {
  1103. struct cgroup *task_cgrp;
  1104. struct cgroup *mem_cgrp;
  1105. /*
  1106. * Need a buffer in BSS, can't rely on allocations. The code relies
  1107. * on the assumption that OOM is serialized for memory controller.
  1108. * If this assumption is broken, revisit this code.
  1109. */
  1110. static char memcg_name[PATH_MAX];
  1111. int ret;
  1112. if (!memcg || !p)
  1113. return;
  1114. rcu_read_lock();
  1115. mem_cgrp = memcg->css.cgroup;
  1116. task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);
  1117. ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
  1118. if (ret < 0) {
  1119. /*
  1120. * Unfortunately, we are unable to convert to a useful name
  1121. * But we'll still print out the usage information
  1122. */
  1123. rcu_read_unlock();
  1124. goto done;
  1125. }
  1126. rcu_read_unlock();
  1127. printk(KERN_INFO "Task in %s killed", memcg_name);
  1128. rcu_read_lock();
  1129. ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
  1130. if (ret < 0) {
  1131. rcu_read_unlock();
  1132. goto done;
  1133. }
  1134. rcu_read_unlock();
  1135. /*
  1136. * Continues from above, so we don't need an KERN_ level
  1137. */
  1138. printk(KERN_CONT " as a result of limit of %s\n", memcg_name);
  1139. done:
  1140. printk(KERN_INFO "memory: usage %llukB, limit %llukB, failcnt %llu\n",
  1141. res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
  1142. res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
  1143. res_counter_read_u64(&memcg->res, RES_FAILCNT));
  1144. printk(KERN_INFO "memory+swap: usage %llukB, limit %llukB, "
  1145. "failcnt %llu\n",
  1146. res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
  1147. res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
  1148. res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
  1149. }
  1150. /*
  1151. * This function returns the number of memcg under hierarchy tree. Returns
  1152. * 1(self count) if no children.
  1153. */
  1154. static int mem_cgroup_count_children(struct mem_cgroup *mem)
  1155. {
  1156. int num = 0;
  1157. struct mem_cgroup *iter;
  1158. for_each_mem_cgroup_tree(iter, mem)
  1159. num++;
  1160. return num;
  1161. }
  1162. /*
  1163. * Return the memory (and swap, if configured) limit for a memcg.
  1164. */
  1165. u64 mem_cgroup_get_limit(struct mem_cgroup *memcg)
  1166. {
  1167. u64 limit;
  1168. u64 memsw;
  1169. limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
  1170. limit += total_swap_pages << PAGE_SHIFT;
  1171. memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
  1172. /*
  1173. * If memsw is finite and limits the amount of swap space available
  1174. * to this memcg, return that limit.
  1175. */
  1176. return min(limit, memsw);
  1177. }
  1178. /*
  1179. * Visit the first child (need not be the first child as per the ordering
  1180. * of the cgroup list, since we track last_scanned_child) of @mem and use
  1181. * that to reclaim free pages from.
  1182. */
  1183. static struct mem_cgroup *
  1184. mem_cgroup_select_victim(struct mem_cgroup *root_mem)
  1185. {
  1186. struct mem_cgroup *ret = NULL;
  1187. struct cgroup_subsys_state *css;
  1188. int nextid, found;
  1189. if (!root_mem->use_hierarchy) {
  1190. css_get(&root_mem->css);
  1191. ret = root_mem;
  1192. }
  1193. while (!ret) {
  1194. rcu_read_lock();
  1195. nextid = root_mem->last_scanned_child + 1;
  1196. css = css_get_next(&mem_cgroup_subsys, nextid, &root_mem->css,
  1197. &found);
  1198. if (css && css_tryget(css))
  1199. ret = container_of(css, struct mem_cgroup, css);
  1200. rcu_read_unlock();
  1201. /* Updates scanning parameter */
  1202. spin_lock(&root_mem->reclaim_param_lock);
  1203. if (!css) {
  1204. /* this means start scan from ID:1 */
  1205. root_mem->last_scanned_child = 0;
  1206. } else
  1207. root_mem->last_scanned_child = found;
  1208. spin_unlock(&root_mem->reclaim_param_lock);
  1209. }
  1210. return ret;
  1211. }
  1212. /*
  1213. * Scan the hierarchy if needed to reclaim memory. We remember the last child
  1214. * we reclaimed from, so that we don't end up penalizing one child extensively
  1215. * based on its position in the children list.
  1216. *
  1217. * root_mem is the original ancestor that we've been reclaim from.
  1218. *
  1219. * We give up and return to the caller when we visit root_mem twice.
  1220. * (other groups can be removed while we're walking....)
  1221. *
  1222. * If shrink==true, for avoiding to free too much, this returns immedieately.
  1223. */
  1224. static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
  1225. struct zone *zone,
  1226. gfp_t gfp_mask,
  1227. unsigned long reclaim_options)
  1228. {
  1229. struct mem_cgroup *victim;
  1230. int ret, total = 0;
  1231. int loop = 0;
  1232. bool noswap = reclaim_options & MEM_CGROUP_RECLAIM_NOSWAP;
  1233. bool shrink = reclaim_options & MEM_CGROUP_RECLAIM_SHRINK;
  1234. bool check_soft = reclaim_options & MEM_CGROUP_RECLAIM_SOFT;
  1235. unsigned long excess = mem_cgroup_get_excess(root_mem);
  1236. /* If memsw_is_minimum==1, swap-out is of-no-use. */
  1237. if (root_mem->memsw_is_minimum)
  1238. noswap = true;
  1239. while (1) {
  1240. victim = mem_cgroup_select_victim(root_mem);
  1241. if (victim == root_mem) {
  1242. loop++;
  1243. if (loop >= 1)
  1244. drain_all_stock_async();
  1245. if (loop >= 2) {
  1246. /*
  1247. * If we have not been able to reclaim
  1248. * anything, it might because there are
  1249. * no reclaimable pages under this hierarchy
  1250. */
  1251. if (!check_soft || !total) {
  1252. css_put(&victim->css);
  1253. break;
  1254. }
  1255. /*
  1256. * We want to do more targetted reclaim.
  1257. * excess >> 2 is not to excessive so as to
  1258. * reclaim too much, nor too less that we keep
  1259. * coming back to reclaim from this cgroup
  1260. */
  1261. if (total >= (excess >> 2) ||
  1262. (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) {
  1263. css_put(&victim->css);
  1264. break;
  1265. }
  1266. }
  1267. }
  1268. if (!mem_cgroup_local_usage(victim)) {
  1269. /* this cgroup's local usage == 0 */
  1270. css_put(&victim->css);
  1271. continue;
  1272. }
  1273. /* we use swappiness of local cgroup */
  1274. if (check_soft)
  1275. ret = mem_cgroup_shrink_node_zone(victim, gfp_mask,
  1276. noswap, get_swappiness(victim), zone);
  1277. else
  1278. ret = try_to_free_mem_cgroup_pages(victim, gfp_mask,
  1279. noswap, get_swappiness(victim));
  1280. css_put(&victim->css);
  1281. /*
  1282. * At shrinking usage, we can't check we should stop here or
  1283. * reclaim more. It's depends on callers. last_scanned_child
  1284. * will work enough for keeping fairness under tree.
  1285. */
  1286. if (shrink)
  1287. return ret;
  1288. total += ret;
  1289. if (check_soft) {
  1290. if (res_counter_check_under_soft_limit(&root_mem->res))
  1291. return total;
  1292. } else if (mem_cgroup_check_under_limit(root_mem))
  1293. return 1 + total;
  1294. }
  1295. return total;
  1296. }
  1297. /*
  1298. * Check OOM-Killer is already running under our hierarchy.
  1299. * If someone is running, return false.
  1300. */
  1301. static bool mem_cgroup_oom_lock(struct mem_cgroup *mem)
  1302. {
  1303. int x, lock_count = 0;
  1304. struct mem_cgroup *iter;
  1305. for_each_mem_cgroup_tree(iter, mem) {
  1306. x = atomic_inc_return(&iter->oom_lock);
  1307. lock_count = max(x, lock_count);
  1308. }
  1309. if (lock_count == 1)
  1310. return true;
  1311. return false;
  1312. }
  1313. static int mem_cgroup_oom_unlock(struct mem_cgroup *mem)
  1314. {
  1315. struct mem_cgroup *iter;
  1316. /*
  1317. * When a new child is created while the hierarchy is under oom,
  1318. * mem_cgroup_oom_lock() may not be called. We have to use
  1319. * atomic_add_unless() here.
  1320. */
  1321. for_each_mem_cgroup_tree(iter, mem)
  1322. atomic_add_unless(&iter->oom_lock, -1, 0);
  1323. return 0;
  1324. }
  1325. static DEFINE_MUTEX(memcg_oom_mutex);
  1326. static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
  1327. struct oom_wait_info {
  1328. struct mem_cgroup *mem;
  1329. wait_queue_t wait;
  1330. };
  1331. static int memcg_oom_wake_function(wait_queue_t *wait,
  1332. unsigned mode, int sync, void *arg)
  1333. {
  1334. struct mem_cgroup *wake_mem = (struct mem_cgroup *)arg;
  1335. struct oom_wait_info *oom_wait_info;
  1336. oom_wait_info = container_of(wait, struct oom_wait_info, wait);
  1337. if (oom_wait_info->mem == wake_mem)
  1338. goto wakeup;
  1339. /* if no hierarchy, no match */
  1340. if (!oom_wait_info->mem->use_hierarchy || !wake_mem->use_hierarchy)
  1341. return 0;
  1342. /*
  1343. * Both of oom_wait_info->mem and wake_mem are stable under us.
  1344. * Then we can use css_is_ancestor without taking care of RCU.
  1345. */
  1346. if (!css_is_ancestor(&oom_wait_info->mem->css, &wake_mem->css) &&
  1347. !css_is_ancestor(&wake_mem->css, &oom_wait_info->mem->css))
  1348. return 0;
  1349. wakeup:
  1350. return autoremove_wake_function(wait, mode, sync, arg);
  1351. }
  1352. static void memcg_wakeup_oom(struct mem_cgroup *mem)
  1353. {
  1354. /* for filtering, pass "mem" as argument. */
  1355. __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, mem);
  1356. }
  1357. static void memcg_oom_recover(struct mem_cgroup *mem)
  1358. {
  1359. if (mem && atomic_read(&mem->oom_lock))
  1360. memcg_wakeup_oom(mem);
  1361. }
  1362. /*
  1363. * try to call OOM killer. returns false if we should exit memory-reclaim loop.
  1364. */
  1365. bool mem_cgroup_handle_oom(struct mem_cgroup *mem, gfp_t mask)
  1366. {
  1367. struct oom_wait_info owait;
  1368. bool locked, need_to_kill;
  1369. owait.mem = mem;
  1370. owait.wait.flags = 0;
  1371. owait.wait.func = memcg_oom_wake_function;
  1372. owait.wait.private = current;
  1373. INIT_LIST_HEAD(&owait.wait.task_list);
  1374. need_to_kill = true;
  1375. /* At first, try to OOM lock hierarchy under mem.*/
  1376. mutex_lock(&memcg_oom_mutex);
  1377. locked = mem_cgroup_oom_lock(mem);
  1378. /*
  1379. * Even if signal_pending(), we can't quit charge() loop without
  1380. * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL
  1381. * under OOM is always welcomed, use TASK_KILLABLE here.
  1382. */
  1383. prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
  1384. if (!locked || mem->oom_kill_disable)
  1385. need_to_kill = false;
  1386. if (locked)
  1387. mem_cgroup_oom_notify(mem);
  1388. mutex_unlock(&memcg_oom_mutex);
  1389. if (need_to_kill) {
  1390. finish_wait(&memcg_oom_waitq, &owait.wait);
  1391. mem_cgroup_out_of_memory(mem, mask);
  1392. } else {
  1393. schedule();
  1394. finish_wait(&memcg_oom_waitq, &owait.wait);
  1395. }
  1396. mutex_lock(&memcg_oom_mutex);
  1397. mem_cgroup_oom_unlock(mem);
  1398. memcg_wakeup_oom(mem);
  1399. mutex_unlock(&memcg_oom_mutex);
  1400. if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current))
  1401. return false;
  1402. /* Give chance to dying process */
  1403. schedule_timeout(1);
  1404. return true;
  1405. }
  1406. /*
  1407. * Currently used to update mapped file statistics, but the routine can be
  1408. * generalized to update other statistics as well.
  1409. *
  1410. * Notes: Race condition
  1411. *
  1412. * We usually use page_cgroup_lock() for accessing page_cgroup member but
  1413. * it tends to be costly. But considering some conditions, we doesn't need
  1414. * to do so _always_.
  1415. *
  1416. * Considering "charge", lock_page_cgroup() is not required because all
  1417. * file-stat operations happen after a page is attached to radix-tree. There
  1418. * are no race with "charge".
  1419. *
  1420. * Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup
  1421. * at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even
  1422. * if there are race with "uncharge". Statistics itself is properly handled
  1423. * by flags.
  1424. *
  1425. * Considering "move", this is an only case we see a race. To make the race
  1426. * small, we check MEM_CGROUP_ON_MOVE percpu value and detect there are
  1427. * possibility of race condition. If there is, we take a lock.
  1428. */
  1429. void mem_cgroup_update_page_stat(struct page *page,
  1430. enum mem_cgroup_page_stat_item idx, int val)
  1431. {
  1432. struct mem_cgroup *mem;
  1433. struct page_cgroup *pc = lookup_page_cgroup(page);
  1434. bool need_unlock = false;
  1435. unsigned long uninitialized_var(flags);
  1436. if (unlikely(!pc))
  1437. return;
  1438. rcu_read_lock();
  1439. mem = pc->mem_cgroup;
  1440. if (unlikely(!mem || !PageCgroupUsed(pc)))
  1441. goto out;
  1442. /* pc->mem_cgroup is unstable ? */
  1443. if (unlikely(mem_cgroup_stealed(mem)) || PageTransHuge(page)) {
  1444. /* take a lock against to access pc->mem_cgroup */
  1445. move_lock_page_cgroup(pc, &flags);
  1446. need_unlock = true;
  1447. mem = pc->mem_cgroup;
  1448. if (!mem || !PageCgroupUsed(pc))
  1449. goto out;
  1450. }
  1451. switch (idx) {
  1452. case MEMCG_NR_FILE_MAPPED:
  1453. if (val > 0)
  1454. SetPageCgroupFileMapped(pc);
  1455. else if (!page_mapped(page))
  1456. ClearPageCgroupFileMapped(pc);
  1457. idx = MEM_CGROUP_STAT_FILE_MAPPED;
  1458. break;
  1459. default:
  1460. BUG();
  1461. }
  1462. this_cpu_add(mem->stat->count[idx], val);
  1463. out:
  1464. if (unlikely(need_unlock))
  1465. move_unlock_page_cgroup(pc, &flags);
  1466. rcu_read_unlock();
  1467. return;
  1468. }
  1469. EXPORT_SYMBOL(mem_cgroup_update_page_stat);
  1470. /*
  1471. * size of first charge trial. "32" comes from vmscan.c's magic value.
  1472. * TODO: maybe necessary to use big numbers in big irons.
  1473. */
  1474. #define CHARGE_SIZE (32 * PAGE_SIZE)
  1475. struct memcg_stock_pcp {
  1476. struct mem_cgroup *cached; /* this never be root cgroup */
  1477. int charge;
  1478. struct work_struct work;
  1479. };
  1480. static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
  1481. static atomic_t memcg_drain_count;
  1482. /*
  1483. * Try to consume stocked charge on this cpu. If success, PAGE_SIZE is consumed
  1484. * from local stock and true is returned. If the stock is 0 or charges from a
  1485. * cgroup which is not current target, returns false. This stock will be
  1486. * refilled.
  1487. */
  1488. static bool consume_stock(struct mem_cgroup *mem)
  1489. {
  1490. struct memcg_stock_pcp *stock;
  1491. bool ret = true;
  1492. stock = &get_cpu_var(memcg_stock);
  1493. if (mem == stock->cached && stock->charge)
  1494. stock->charge -= PAGE_SIZE;
  1495. else /* need to call res_counter_charge */
  1496. ret = false;
  1497. put_cpu_var(memcg_stock);
  1498. return ret;
  1499. }
  1500. /*
  1501. * Returns stocks cached in percpu to res_counter and reset cached information.
  1502. */
  1503. static void drain_stock(struct memcg_stock_pcp *stock)
  1504. {
  1505. struct mem_cgroup *old = stock->cached;
  1506. if (stock->charge) {
  1507. res_counter_uncharge(&old->res, stock->charge);
  1508. if (do_swap_account)
  1509. res_counter_uncharge(&old->memsw, stock->charge);
  1510. }
  1511. stock->cached = NULL;
  1512. stock->charge = 0;
  1513. }
  1514. /*
  1515. * This must be called under preempt disabled or must be called by
  1516. * a thread which is pinned to local cpu.
  1517. */
  1518. static void drain_local_stock(struct work_struct *dummy)
  1519. {
  1520. struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
  1521. drain_stock(stock);
  1522. }
  1523. /*
  1524. * Cache charges(val) which is from res_counter, to local per_cpu area.
  1525. * This will be consumed by consume_stock() function, later.
  1526. */
  1527. static void refill_stock(struct mem_cgroup *mem, int val)
  1528. {
  1529. struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
  1530. if (stock->cached != mem) { /* reset if necessary */
  1531. drain_stock(stock);
  1532. stock->cached = mem;
  1533. }
  1534. stock->charge += val;
  1535. put_cpu_var(memcg_stock);
  1536. }
  1537. /*
  1538. * Tries to drain stocked charges in other cpus. This function is asynchronous
  1539. * and just put a work per cpu for draining localy on each cpu. Caller can
  1540. * expects some charges will be back to res_counter later but cannot wait for
  1541. * it.
  1542. */
  1543. static void drain_all_stock_async(void)
  1544. {
  1545. int cpu;
  1546. /* This function is for scheduling "drain" in asynchronous way.
  1547. * The result of "drain" is not directly handled by callers. Then,
  1548. * if someone is calling drain, we don't have to call drain more.
  1549. * Anyway, WORK_STRUCT_PENDING check in queue_work_on() will catch if
  1550. * there is a race. We just do loose check here.
  1551. */
  1552. if (atomic_read(&memcg_drain_count))
  1553. return;
  1554. /* Notify other cpus that system-wide "drain" is running */
  1555. atomic_inc(&memcg_drain_count);
  1556. get_online_cpus();
  1557. for_each_online_cpu(cpu) {
  1558. struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
  1559. schedule_work_on(cpu, &stock->work);
  1560. }
  1561. put_online_cpus();
  1562. atomic_dec(&memcg_drain_count);
  1563. /* We don't wait for flush_work */
  1564. }
  1565. /* This is a synchronous drain interface. */
  1566. static void drain_all_stock_sync(void)
  1567. {
  1568. /* called when force_empty is called */
  1569. atomic_inc(&memcg_drain_count);
  1570. schedule_on_each_cpu(drain_local_stock);
  1571. atomic_dec(&memcg_drain_count);
  1572. }
  1573. /*
  1574. * This function drains percpu counter value from DEAD cpu and
  1575. * move it to local cpu. Note that this function can be preempted.
  1576. */
  1577. static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *mem, int cpu)
  1578. {
  1579. int i;
  1580. spin_lock(&mem->pcp_counter_lock);
  1581. for (i = 0; i < MEM_CGROUP_STAT_DATA; i++) {
  1582. s64 x = per_cpu(mem->stat->count[i], cpu);
  1583. per_cpu(mem->stat->count[i], cpu) = 0;
  1584. mem->nocpu_base.count[i] += x;
  1585. }
  1586. /* need to clear ON_MOVE value, works as a kind of lock. */
  1587. per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) = 0;
  1588. spin_unlock(&mem->pcp_counter_lock);
  1589. }
  1590. static void synchronize_mem_cgroup_on_move(struct mem_cgroup *mem, int cpu)
  1591. {
  1592. int idx = MEM_CGROUP_ON_MOVE;
  1593. spin_lock(&mem->pcp_counter_lock);
  1594. per_cpu(mem->stat->count[idx], cpu) = mem->nocpu_base.count[idx];
  1595. spin_unlock(&mem->pcp_counter_lock);
  1596. }
  1597. static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb,
  1598. unsigned long action,
  1599. void *hcpu)
  1600. {
  1601. int cpu = (unsigned long)hcpu;
  1602. struct memcg_stock_pcp *stock;
  1603. struct mem_cgroup *iter;
  1604. if ((action == CPU_ONLINE)) {
  1605. for_each_mem_cgroup_all(iter)
  1606. synchronize_mem_cgroup_on_move(iter, cpu);
  1607. return NOTIFY_OK;
  1608. }
  1609. if ((action != CPU_DEAD) || action != CPU_DEAD_FROZEN)
  1610. return NOTIFY_OK;
  1611. for_each_mem_cgroup_all(iter)
  1612. mem_cgroup_drain_pcp_counter(iter, cpu);
  1613. stock = &per_cpu(memcg_stock, cpu);
  1614. drain_stock(stock);
  1615. return NOTIFY_OK;
  1616. }
  1617. /* See __mem_cgroup_try_charge() for details */
  1618. enum {
  1619. CHARGE_OK, /* success */
  1620. CHARGE_RETRY, /* need to retry but retry is not bad */
  1621. CHARGE_NOMEM, /* we can't do more. return -ENOMEM */
  1622. CHARGE_WOULDBLOCK, /* GFP_WAIT wasn't set and no enough res. */
  1623. CHARGE_OOM_DIE, /* the current is killed because of OOM */
  1624. };
  1625. static int __mem_cgroup_do_charge(struct mem_cgroup *mem, gfp_t gfp_mask,
  1626. int csize, bool oom_check)
  1627. {
  1628. struct mem_cgroup *mem_over_limit;
  1629. struct res_counter *fail_res;
  1630. unsigned long flags = 0;
  1631. int ret;
  1632. ret = res_counter_charge(&mem->res, csize, &fail_res);
  1633. if (likely(!ret)) {
  1634. if (!do_swap_account)
  1635. return CHARGE_OK;
  1636. ret = res_counter_charge(&mem->memsw, csize, &fail_res);
  1637. if (likely(!ret))
  1638. return CHARGE_OK;
  1639. res_counter_uncharge(&mem->res, csize);
  1640. mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw);
  1641. flags |= MEM_CGROUP_RECLAIM_NOSWAP;
  1642. } else
  1643. mem_over_limit = mem_cgroup_from_res_counter(fail_res, res);
  1644. /*
  1645. * csize can be either a huge page (HPAGE_SIZE), a batch of
  1646. * regular pages (CHARGE_SIZE), or a single regular page
  1647. * (PAGE_SIZE).
  1648. *
  1649. * Never reclaim on behalf of optional batching, retry with a
  1650. * single page instead.
  1651. */
  1652. if (csize == CHARGE_SIZE)
  1653. return CHARGE_RETRY;
  1654. if (!(gfp_mask & __GFP_WAIT))
  1655. return CHARGE_WOULDBLOCK;
  1656. ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, NULL,
  1657. gfp_mask, flags);
  1658. if (mem_cgroup_check_margin(mem_over_limit, csize))
  1659. return CHARGE_RETRY;
  1660. /*
  1661. * Even though the limit is exceeded at this point, reclaim
  1662. * may have been able to free some pages. Retry the charge
  1663. * before killing the task.
  1664. *
  1665. * Only for regular pages, though: huge pages are rather
  1666. * unlikely to succeed so close to the limit, and we fall back
  1667. * to regular pages anyway in case of failure.
  1668. */
  1669. if (csize == PAGE_SIZE && ret)
  1670. return CHARGE_RETRY;
  1671. /*
  1672. * At task move, charge accounts can be doubly counted. So, it's
  1673. * better to wait until the end of task_move if something is going on.
  1674. */
  1675. if (mem_cgroup_wait_acct_move(mem_over_limit))
  1676. return CHARGE_RETRY;
  1677. /* If we don't need to call oom-killer at el, return immediately */
  1678. if (!oom_check)
  1679. return CHARGE_NOMEM;
  1680. /* check OOM */
  1681. if (!mem_cgroup_handle_oom(mem_over_limit, gfp_mask))
  1682. return CHARGE_OOM_DIE;
  1683. return CHARGE_RETRY;
  1684. }
  1685. /*
  1686. * Unlike exported interface, "oom" parameter is added. if oom==true,
  1687. * oom-killer can be invoked.
  1688. */
  1689. static int __mem_cgroup_try_charge(struct mm_struct *mm,
  1690. gfp_t gfp_mask,
  1691. struct mem_cgroup **memcg, bool oom,
  1692. int page_size)
  1693. {
  1694. int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
  1695. struct mem_cgroup *mem = NULL;
  1696. int ret;
  1697. int csize = max(CHARGE_SIZE, (unsigned long) page_size);
  1698. /*
  1699. * Unlike gloval-vm's OOM-kill, we're not in memory shortage
  1700. * in system level. So, allow to go ahead dying process in addition to
  1701. * MEMDIE process.
  1702. */
  1703. if (unlikely(test_thread_flag(TIF_MEMDIE)
  1704. || fatal_signal_pending(current)))
  1705. goto bypass;
  1706. /*
  1707. * We always charge the cgroup the mm_struct belongs to.
  1708. * The mm_struct's mem_cgroup changes on task migration if the
  1709. * thread group leader migrates. It's possible that mm is not
  1710. * set, if so charge the init_mm (happens for pagecache usage).
  1711. */
  1712. if (!*memcg && !mm)
  1713. goto bypass;
  1714. again:
  1715. if (*memcg) { /* css should be a valid one */
  1716. mem = *memcg;
  1717. VM_BUG_ON(css_is_removed(&mem->css));
  1718. if (mem_cgroup_is_root(mem))
  1719. goto done;
  1720. if (page_size == PAGE_SIZE && consume_stock(mem))
  1721. goto done;
  1722. css_get(&mem->css);
  1723. } else {
  1724. struct task_struct *p;
  1725. rcu_read_lock();
  1726. p = rcu_dereference(mm->owner);
  1727. /*
  1728. * Because we don't have task_lock(), "p" can exit.
  1729. * In that case, "mem" can point to root or p can be NULL with
  1730. * race with swapoff. Then, we have small risk of mis-accouning.
  1731. * But such kind of mis-account by race always happens because
  1732. * we don't have cgroup_mutex(). It's overkill and we allo that
  1733. * small race, here.
  1734. * (*) swapoff at el will charge against mm-struct not against
  1735. * task-struct. So, mm->owner can be NULL.
  1736. */
  1737. mem = mem_cgroup_from_task(p);
  1738. if (!mem || mem_cgroup_is_root(mem)) {
  1739. rcu_read_unlock();
  1740. goto done;
  1741. }
  1742. if (page_size == PAGE_SIZE && consume_stock(mem)) {
  1743. /*
  1744. * It seems dagerous to access memcg without css_get().
  1745. * But considering how consume_stok works, it's not
  1746. * necessary. If consume_stock success, some charges
  1747. * from this memcg are cached on this cpu. So, we
  1748. * don't need to call css_get()/css_tryget() before
  1749. * calling consume_stock().
  1750. */
  1751. rcu_read_unlock();
  1752. goto done;
  1753. }
  1754. /* after here, we may be blocked. we need to get refcnt */
  1755. if (!css_tryget(&mem->css)) {
  1756. rcu_read_unlock();
  1757. goto again;
  1758. }
  1759. rcu_read_unlock();
  1760. }
  1761. do {
  1762. bool oom_check;
  1763. /* If killed, bypass charge */
  1764. if (fatal_signal_pending(current)) {
  1765. css_put(&mem->css);
  1766. goto bypass;
  1767. }
  1768. oom_check = false;
  1769. if (oom && !nr_oom_retries) {
  1770. oom_check = true;
  1771. nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
  1772. }
  1773. ret = __mem_cgroup_do_charge(mem, gfp_mask, csize, oom_check);
  1774. switch (ret) {
  1775. case CHARGE_OK:
  1776. break;
  1777. case CHARGE_RETRY: /* not in OOM situation but retry */
  1778. csize = page_size;
  1779. css_put(&mem->css);
  1780. mem = NULL;
  1781. goto again;
  1782. case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */
  1783. css_put(&mem->css);
  1784. goto nomem;
  1785. case CHARGE_NOMEM: /* OOM routine works */
  1786. if (!oom) {
  1787. css_put(&mem->css);
  1788. goto nomem;
  1789. }
  1790. /* If oom, we never return -ENOMEM */
  1791. nr_oom_retries--;
  1792. break;
  1793. case CHARGE_OOM_DIE: /* Killed by OOM Killer */
  1794. css_put(&mem->css);
  1795. goto bypass;
  1796. }
  1797. } while (ret != CHARGE_OK);
  1798. if (csize > page_size)
  1799. refill_stock(mem, csize - page_size);
  1800. css_put(&mem->css);
  1801. done:
  1802. *memcg = mem;
  1803. return 0;
  1804. nomem:
  1805. *memcg = NULL;
  1806. return -ENOMEM;
  1807. bypass:
  1808. *memcg = NULL;
  1809. return 0;
  1810. }
  1811. /*
  1812. * Somemtimes we have to undo a charge we got by try_charge().
  1813. * This function is for that and do uncharge, put css's refcnt.
  1814. * gotten by try_charge().
  1815. */
  1816. static void __mem_cgroup_cancel_charge(struct mem_cgroup *mem,
  1817. unsigned long count)
  1818. {
  1819. if (!mem_cgroup_is_root(mem)) {
  1820. res_counter_uncharge(&mem->res, PAGE_SIZE * count);
  1821. if (do_swap_account)
  1822. res_counter_uncharge(&mem->memsw, PAGE_SIZE * count);
  1823. }
  1824. }
  1825. static void mem_cgroup_cancel_charge(struct mem_cgroup *mem,
  1826. int page_size)
  1827. {
  1828. __mem_cgroup_cancel_charge(mem, page_size >> PAGE_SHIFT);
  1829. }
  1830. /*
  1831. * A helper function to get mem_cgroup from ID. must be called under
  1832. * rcu_read_lock(). The caller must check css_is_removed() or some if
  1833. * it's concern. (dropping refcnt from swap can be called against removed
  1834. * memcg.)
  1835. */
  1836. static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
  1837. {
  1838. struct cgroup_subsys_state *css;
  1839. /* ID 0 is unused ID */
  1840. if (!id)
  1841. return NULL;
  1842. css = css_lookup(&mem_cgroup_subsys, id);
  1843. if (!css)
  1844. return NULL;
  1845. return container_of(css, struct mem_cgroup, css);
  1846. }
  1847. struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
  1848. {
  1849. struct mem_cgroup *mem = NULL;
  1850. struct page_cgroup *pc;
  1851. unsigned short id;
  1852. swp_entry_t ent;
  1853. VM_BUG_ON(!PageLocked(page));
  1854. pc = lookup_page_cgroup(page);
  1855. lock_page_cgroup(pc);
  1856. if (PageCgroupUsed(pc)) {
  1857. mem = pc->mem_cgroup;
  1858. if (mem && !css_tryget(&mem->css))
  1859. mem = NULL;
  1860. } else if (PageSwapCache(page)) {
  1861. ent.val = page_private(page);
  1862. id = lookup_swap_cgroup(ent);
  1863. rcu_read_lock();
  1864. mem = mem_cgroup_lookup(id);
  1865. if (mem && !css_tryget(&mem->css))
  1866. mem = NULL;
  1867. rcu_read_unlock();
  1868. }
  1869. unlock_page_cgroup(pc);
  1870. return mem;
  1871. }
  1872. static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
  1873. struct page_cgroup *pc,
  1874. enum charge_type ctype,
  1875. int page_size)
  1876. {
  1877. int nr_pages = page_size >> PAGE_SHIFT;
  1878. /* try_charge() can return NULL to *memcg, taking care of it. */
  1879. if (!mem)
  1880. return;
  1881. lock_page_cgroup(pc);
  1882. if (unlikely(PageCgroupUsed(pc))) {
  1883. unlock_page_cgroup(pc);
  1884. mem_cgroup_cancel_charge(mem, page_size);
  1885. return;
  1886. }
  1887. /*
  1888. * we don't need page_cgroup_lock about tail pages, becase they are not
  1889. * accessed by any other context at this point.
  1890. */
  1891. pc->mem_cgroup = mem;
  1892. /*
  1893. * We access a page_cgroup asynchronously without lock_page_cgroup().
  1894. * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
  1895. * is accessed after testing USED bit. To make pc->mem_cgroup visible
  1896. * before USED bit, we need memory barrier here.
  1897. * See mem_cgroup_add_lru_list(), etc.
  1898. */
  1899. smp_wmb();
  1900. switch (ctype) {
  1901. case MEM_CGROUP_CHARGE_TYPE_CACHE:
  1902. case MEM_CGROUP_CHARGE_TYPE_SHMEM:
  1903. SetPageCgroupCache(pc);
  1904. SetPageCgroupUsed(pc);
  1905. break;
  1906. case MEM_CGROUP_CHARGE_TYPE_MAPPED:
  1907. ClearPageCgroupCache(pc);
  1908. SetPageCgroupUsed(pc);
  1909. break;
  1910. default:
  1911. break;
  1912. }
  1913. mem_cgroup_charge_statistics(mem, PageCgroupCache(pc), nr_pages);
  1914. unlock_page_cgroup(pc);
  1915. /*
  1916. * "charge_statistics" updated event counter. Then, check it.
  1917. * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
  1918. * if they exceeds softlimit.
  1919. */
  1920. memcg_check_events(mem, pc->page);
  1921. }
  1922. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  1923. #define PCGF_NOCOPY_AT_SPLIT ((1 << PCG_LOCK) | (1 << PCG_MOVE_LOCK) |\
  1924. (1 << PCG_ACCT_LRU) | (1 << PCG_MIGRATION))
  1925. /*
  1926. * Because tail pages are not marked as "used", set it. We're under
  1927. * zone->lru_lock, 'splitting on pmd' and compund_lock.
  1928. */
  1929. void mem_cgroup_split_huge_fixup(struct page *head, struct page *tail)
  1930. {
  1931. struct page_cgroup *head_pc = lookup_page_cgroup(head);
  1932. struct page_cgroup *tail_pc = lookup_page_cgroup(tail);
  1933. unsigned long flags;
  1934. if (mem_cgroup_disabled())
  1935. return;
  1936. /*
  1937. * We have no races with charge/uncharge but will have races with
  1938. * page state accounting.
  1939. */
  1940. move_lock_page_cgroup(head_pc, &flags);
  1941. tail_pc->mem_cgroup = head_pc->mem_cgroup;
  1942. smp_wmb(); /* see __commit_charge() */
  1943. if (PageCgroupAcctLRU(head_pc)) {
  1944. enum lru_list lru;
  1945. struct mem_cgroup_per_zone *mz;
  1946. /*
  1947. * LRU flags cannot be copied because we need to add tail
  1948. *.page to LRU by generic call and our hook will be called.
  1949. * We hold lru_lock, then, reduce counter directly.
  1950. */
  1951. lru = page_lru(head);
  1952. mz = page_cgroup_zoneinfo(head_pc);
  1953. MEM_CGROUP_ZSTAT(mz, lru) -= 1;
  1954. }
  1955. tail_pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT;
  1956. move_unlock_page_cgroup(head_pc, &flags);
  1957. }
  1958. #endif
  1959. /**
  1960. * __mem_cgroup_move_account - move account of the page
  1961. * @pc: page_cgroup of the page.
  1962. * @from: mem_cgroup which the page is moved from.
  1963. * @to: mem_cgroup which the page is moved to. @from != @to.
  1964. * @uncharge: whether we should call uncharge and css_put against @from.
  1965. *
  1966. * The caller must confirm following.
  1967. * - page is not on LRU (isolate_page() is useful.)
  1968. * - the pc is locked, used, and ->mem_cgroup points to @from.
  1969. *
  1970. * This function doesn't do "charge" nor css_get to new cgroup. It should be
  1971. * done by a caller(__mem_cgroup_try_charge would be usefull). If @uncharge is
  1972. * true, this function does "uncharge" from old cgroup, but it doesn't if
  1973. * @uncharge is false, so a caller should do "uncharge".
  1974. */
  1975. static void __mem_cgroup_move_account(struct page_cgroup *pc,
  1976. struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge,
  1977. int charge_size)
  1978. {
  1979. int nr_pages = charge_size >> PAGE_SHIFT;
  1980. VM_BUG_ON(from == to);
  1981. VM_BUG_ON(PageLRU(pc->page));
  1982. VM_BUG_ON(!page_is_cgroup_locked(pc));
  1983. VM_BUG_ON(!PageCgroupUsed(pc));
  1984. VM_BUG_ON(pc->mem_cgroup != from);
  1985. if (PageCgroupFileMapped(pc)) {
  1986. /* Update mapped_file data for mem_cgroup */
  1987. preempt_disable();
  1988. __this_cpu_dec(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
  1989. __this_cpu_inc(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
  1990. preempt_enable();
  1991. }
  1992. mem_cgroup_charge_statistics(from, PageCgroupCache(pc), -nr_pages);
  1993. if (uncharge)
  1994. /* This is not "cancel", but cancel_charge does all we need. */
  1995. mem_cgroup_cancel_charge(from, charge_size);
  1996. /* caller should have done css_get */
  1997. pc->mem_cgroup = to;
  1998. mem_cgroup_charge_statistics(to, PageCgroupCache(pc), nr_pages);
  1999. /*
  2000. * We charges against "to" which may not have any tasks. Then, "to"
  2001. * can be under rmdir(). But in current implementation, caller of
  2002. * this function is just force_empty() and move charge, so it's
  2003. * garanteed that "to" is never removed. So, we don't check rmdir
  2004. * status here.
  2005. */
  2006. }
  2007. /*
  2008. * check whether the @pc is valid for moving account and call
  2009. * __mem_cgroup_move_account()
  2010. */
  2011. static int mem_cgroup_move_account(struct page_cgroup *pc,
  2012. struct mem_cgroup *from, struct mem_cgroup *to,
  2013. bool uncharge, int charge_size)
  2014. {
  2015. int ret = -EINVAL;
  2016. unsigned long flags;
  2017. /*
  2018. * The page is isolated from LRU. So, collapse function
  2019. * will not handle this page. But page splitting can happen.
  2020. * Do this check under compound_page_lock(). The caller should
  2021. * hold it.
  2022. */
  2023. if ((charge_size > PAGE_SIZE) && !PageTransHuge(pc->page))
  2024. return -EBUSY;
  2025. lock_page_cgroup(pc);
  2026. if (PageCgroupUsed(pc) && pc->mem_cgroup == from) {
  2027. move_lock_page_cgroup(pc, &flags);
  2028. __mem_cgroup_move_account(pc, from, to, uncharge, charge_size);
  2029. move_unlock_page_cgroup(pc, &flags);
  2030. ret = 0;
  2031. }
  2032. unlock_page_cgroup(pc);
  2033. /*
  2034. * check events
  2035. */
  2036. memcg_check_events(to, pc->page);
  2037. memcg_check_events(from, pc->page);
  2038. return ret;
  2039. }
  2040. /*
  2041. * move charges to its parent.
  2042. */
  2043. static int mem_cgroup_move_parent(struct page_cgroup *pc,
  2044. struct mem_cgroup *child,
  2045. gfp_t gfp_mask)
  2046. {
  2047. struct page *page = pc->page;
  2048. struct cgroup *cg = child->css.cgroup;
  2049. struct cgroup *pcg = cg->parent;
  2050. struct mem_cgroup *parent;
  2051. int page_size = PAGE_SIZE;
  2052. unsigned long flags;
  2053. int ret;
  2054. /* Is ROOT ? */
  2055. if (!pcg)
  2056. return -EINVAL;
  2057. ret = -EBUSY;
  2058. if (!get_page_unless_zero(page))
  2059. goto out;
  2060. if (isolate_lru_page(page))
  2061. goto put;
  2062. if (PageTransHuge(page))
  2063. page_size = HPAGE_SIZE;
  2064. parent = mem_cgroup_from_cont(pcg);
  2065. ret = __mem_cgroup_try_charge(NULL, gfp_mask,
  2066. &parent, false, page_size);
  2067. if (ret || !parent)
  2068. goto put_back;
  2069. if (page_size > PAGE_SIZE)
  2070. flags = compound_lock_irqsave(page);
  2071. ret = mem_cgroup_move_account(pc, child, parent, true, page_size);
  2072. if (ret)
  2073. mem_cgroup_cancel_charge(parent, page_size);
  2074. if (page_size > PAGE_SIZE)
  2075. compound_unlock_irqrestore(page, flags);
  2076. put_back:
  2077. putback_lru_page(page);
  2078. put:
  2079. put_page(page);
  2080. out:
  2081. return ret;
  2082. }
  2083. /*
  2084. * Charge the memory controller for page usage.
  2085. * Return
  2086. * 0 if the charge was successful
  2087. * < 0 if the cgroup is over its limit
  2088. */
  2089. static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
  2090. gfp_t gfp_mask, enum charge_type ctype)
  2091. {
  2092. struct mem_cgroup *mem = NULL;
  2093. int page_size = PAGE_SIZE;
  2094. struct page_cgroup *pc;
  2095. bool oom = true;
  2096. int ret;
  2097. if (PageTransHuge(page)) {
  2098. page_size <<= compound_order(page);
  2099. VM_BUG_ON(!PageTransHuge(page));
  2100. /*
  2101. * Never OOM-kill a process for a huge page. The
  2102. * fault handler will fall back to regular pages.
  2103. */
  2104. oom = false;
  2105. }
  2106. pc = lookup_page_cgroup(page);
  2107. /* can happen at boot */
  2108. if (unlikely(!pc))
  2109. return 0;
  2110. prefetchw(pc);
  2111. ret = __mem_cgroup_try_charge(mm, gfp_mask, &mem, oom, page_size);
  2112. if (ret || !mem)
  2113. return ret;
  2114. __mem_cgroup_commit_charge(mem, pc, ctype, page_size);
  2115. return 0;
  2116. }
  2117. int mem_cgroup_newpage_charge(struct page *page,
  2118. struct mm_struct *mm, gfp_t gfp_mask)
  2119. {
  2120. if (mem_cgroup_disabled())
  2121. return 0;
  2122. /*
  2123. * If already mapped, we don't have to account.
  2124. * If page cache, page->mapping has address_space.
  2125. * But page->mapping may have out-of-use anon_vma pointer,
  2126. * detecit it by PageAnon() check. newly-mapped-anon's page->mapping
  2127. * is NULL.
  2128. */
  2129. if (page_mapped(page) || (page->mapping && !PageAnon(page)))
  2130. return 0;
  2131. if (unlikely(!mm))
  2132. mm = &init_mm;
  2133. return mem_cgroup_charge_common(page, mm, gfp_mask,
  2134. MEM_CGROUP_CHARGE_TYPE_MAPPED);
  2135. }
  2136. static void
  2137. __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
  2138. enum charge_type ctype);
  2139. int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
  2140. gfp_t gfp_mask)
  2141. {
  2142. int ret;
  2143. if (mem_cgroup_disabled())
  2144. return 0;
  2145. if (PageCompound(page))
  2146. return 0;
  2147. /*
  2148. * Corner case handling. This is called from add_to_page_cache()
  2149. * in usual. But some FS (shmem) precharges this page before calling it
  2150. * and call add_to_page_cache() with GFP_NOWAIT.
  2151. *
  2152. * For GFP_NOWAIT case, the page may be pre-charged before calling
  2153. * add_to_page_cache(). (See shmem.c) check it here and avoid to call
  2154. * charge twice. (It works but has to pay a bit larger cost.)
  2155. * And when the page is SwapCache, it should take swap information
  2156. * into account. This is under lock_page() now.
  2157. */
  2158. if (!(gfp_mask & __GFP_WAIT)) {
  2159. struct page_cgroup *pc;
  2160. pc = lookup_page_cgroup(page);
  2161. if (!pc)
  2162. return 0;
  2163. lock_page_cgroup(pc);
  2164. if (PageCgroupUsed(pc)) {
  2165. unlock_page_cgroup(pc);
  2166. return 0;
  2167. }
  2168. unlock_page_cgroup(pc);
  2169. }
  2170. if (unlikely(!mm))
  2171. mm = &init_mm;
  2172. if (page_is_file_cache(page))
  2173. return mem_cgroup_charge_common(page, mm, gfp_mask,
  2174. MEM_CGROUP_CHARGE_TYPE_CACHE);
  2175. /* shmem */
  2176. if (PageSwapCache(page)) {
  2177. struct mem_cgroup *mem = NULL;
  2178. ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
  2179. if (!ret)
  2180. __mem_cgroup_commit_charge_swapin(page, mem,
  2181. MEM_CGROUP_CHARGE_TYPE_SHMEM);
  2182. } else
  2183. ret = mem_cgroup_charge_common(page, mm, gfp_mask,
  2184. MEM_CGROUP_CHARGE_TYPE_SHMEM);
  2185. return ret;
  2186. }
  2187. /*
  2188. * While swap-in, try_charge -> commit or cancel, the page is locked.
  2189. * And when try_charge() successfully returns, one refcnt to memcg without
  2190. * struct page_cgroup is acquired. This refcnt will be consumed by
  2191. * "commit()" or removed by "cancel()"
  2192. */
  2193. int mem_cgroup_try_charge_swapin(struct mm_struct *mm,
  2194. struct page *page,
  2195. gfp_t mask, struct mem_cgroup **ptr)
  2196. {
  2197. struct mem_cgroup *mem;
  2198. int ret;
  2199. if (mem_cgroup_disabled())
  2200. return 0;
  2201. if (!do_swap_account)
  2202. goto charge_cur_mm;
  2203. /*
  2204. * A racing thread's fault, or swapoff, may have already updated
  2205. * the pte, and even removed page from swap cache: in those cases
  2206. * do_swap_page()'s pte_same() test will fail; but there's also a
  2207. * KSM case which does need to charge the page.
  2208. */
  2209. if (!PageSwapCache(page))
  2210. goto charge_cur_mm;
  2211. mem = try_get_mem_cgroup_from_page(page);
  2212. if (!mem)
  2213. goto charge_cur_mm;
  2214. *ptr = mem;
  2215. ret = __mem_cgroup_try_charge(NULL, mask, ptr, true, PAGE_SIZE);
  2216. css_put(&mem->css);
  2217. return ret;
  2218. charge_cur_mm:
  2219. if (unlikely(!mm))
  2220. mm = &init_mm;
  2221. return __mem_cgroup_try_charge(mm, mask, ptr, true, PAGE_SIZE);
  2222. }
  2223. static void
  2224. __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
  2225. enum charge_type ctype)
  2226. {
  2227. struct page_cgroup *pc;
  2228. if (mem_cgroup_disabled())
  2229. return;
  2230. if (!ptr)
  2231. return;
  2232. cgroup_exclude_rmdir(&ptr->css);
  2233. pc = lookup_page_cgroup(page);
  2234. mem_cgroup_lru_del_before_commit_swapcache(page);
  2235. __mem_cgroup_commit_charge(ptr, pc, ctype, PAGE_SIZE);
  2236. mem_cgroup_lru_add_after_commit_swapcache(page);
  2237. /*
  2238. * Now swap is on-memory. This means this page may be
  2239. * counted both as mem and swap....double count.
  2240. * Fix it by uncharging from memsw. Basically, this SwapCache is stable
  2241. * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
  2242. * may call delete_from_swap_cache() before reach here.
  2243. */
  2244. if (do_swap_account && PageSwapCache(page)) {
  2245. swp_entry_t ent = {.val = page_private(page)};
  2246. unsigned short id;
  2247. struct mem_cgroup *memcg;
  2248. id = swap_cgroup_record(ent, 0);
  2249. rcu_read_lock();
  2250. memcg = mem_cgroup_lookup(id);
  2251. if (memcg) {
  2252. /*
  2253. * This recorded memcg can be obsolete one. So, avoid
  2254. * calling css_tryget
  2255. */
  2256. if (!mem_cgroup_is_root(memcg))
  2257. res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
  2258. mem_cgroup_swap_statistics(memcg, false);
  2259. mem_cgroup_put(memcg);
  2260. }
  2261. rcu_read_unlock();
  2262. }
  2263. /*
  2264. * At swapin, we may charge account against cgroup which has no tasks.
  2265. * So, rmdir()->pre_destroy() can be called while we do this charge.
  2266. * In that case, we need to call pre_destroy() again. check it here.
  2267. */
  2268. cgroup_release_and_wakeup_rmdir(&ptr->css);
  2269. }
  2270. void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr)
  2271. {
  2272. __mem_cgroup_commit_charge_swapin(page, ptr,
  2273. MEM_CGROUP_CHARGE_TYPE_MAPPED);
  2274. }
  2275. void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem)
  2276. {
  2277. if (mem_cgroup_disabled())
  2278. return;
  2279. if (!mem)
  2280. return;
  2281. mem_cgroup_cancel_charge(mem, PAGE_SIZE);
  2282. }
  2283. static void
  2284. __do_uncharge(struct mem_cgroup *mem, const enum charge_type ctype,
  2285. int page_size)
  2286. {
  2287. struct memcg_batch_info *batch = NULL;
  2288. bool uncharge_memsw = true;
  2289. /* If swapout, usage of swap doesn't decrease */
  2290. if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
  2291. uncharge_memsw = false;
  2292. batch = &current->memcg_batch;
  2293. /*
  2294. * In usual, we do css_get() when we remember memcg pointer.
  2295. * But in this case, we keep res->usage until end of a series of
  2296. * uncharges. Then, it's ok to ignore memcg's refcnt.
  2297. */
  2298. if (!batch->memcg)
  2299. batch->memcg = mem;
  2300. /*
  2301. * do_batch > 0 when unmapping pages or inode invalidate/truncate.
  2302. * In those cases, all pages freed continously can be expected to be in
  2303. * the same cgroup and we have chance to coalesce uncharges.
  2304. * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE)
  2305. * because we want to do uncharge as soon as possible.
  2306. */
  2307. if (!batch->do_batch || test_thread_flag(TIF_MEMDIE))
  2308. goto direct_uncharge;
  2309. if (page_size != PAGE_SIZE)
  2310. goto direct_uncharge;
  2311. /*
  2312. * In typical case, batch->memcg == mem. This means we can
  2313. * merge a series of uncharges to an uncharge of res_counter.
  2314. * If not, we uncharge res_counter ony by one.
  2315. */
  2316. if (batch->memcg != mem)
  2317. goto direct_uncharge;
  2318. /* remember freed charge and uncharge it later */
  2319. batch->bytes += PAGE_SIZE;
  2320. if (uncharge_memsw)
  2321. batch->memsw_bytes += PAGE_SIZE;
  2322. return;
  2323. direct_uncharge:
  2324. res_counter_uncharge(&mem->res, page_size);
  2325. if (uncharge_memsw)
  2326. res_counter_uncharge(&mem->memsw, page_size);
  2327. if (unlikely(batch->memcg != mem))
  2328. memcg_oom_recover(mem);
  2329. return;
  2330. }
  2331. /*
  2332. * uncharge if !page_mapped(page)
  2333. */
  2334. static struct mem_cgroup *
  2335. __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
  2336. {
  2337. int count;
  2338. struct page_cgroup *pc;
  2339. struct mem_cgroup *mem = NULL;
  2340. int page_size = PAGE_SIZE;
  2341. if (mem_cgroup_disabled())
  2342. return NULL;
  2343. if (PageSwapCache(page))
  2344. return NULL;
  2345. if (PageTransHuge(page)) {
  2346. page_size <<= compound_order(page);
  2347. VM_BUG_ON(!PageTransHuge(page));
  2348. }
  2349. count = page_size >> PAGE_SHIFT;
  2350. /*
  2351. * Check if our page_cgroup is valid
  2352. */
  2353. pc = lookup_page_cgroup(page);
  2354. if (unlikely(!pc || !PageCgroupUsed(pc)))
  2355. return NULL;
  2356. lock_page_cgroup(pc);
  2357. mem = pc->mem_cgroup;
  2358. if (!PageCgroupUsed(pc))
  2359. goto unlock_out;
  2360. switch (ctype) {
  2361. case MEM_CGROUP_CHARGE_TYPE_MAPPED:
  2362. case MEM_CGROUP_CHARGE_TYPE_DROP:
  2363. /* See mem_cgroup_prepare_migration() */
  2364. if (page_mapped(page) || PageCgroupMigration(pc))
  2365. goto unlock_out;
  2366. break;
  2367. case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
  2368. if (!PageAnon(page)) { /* Shared memory */
  2369. if (page->mapping && !page_is_file_cache(page))
  2370. goto unlock_out;
  2371. } else if (page_mapped(page)) /* Anon */
  2372. goto unlock_out;
  2373. break;
  2374. default:
  2375. break;
  2376. }
  2377. mem_cgroup_charge_statistics(mem, PageCgroupCache(pc), -count);
  2378. ClearPageCgroupUsed(pc);
  2379. /*
  2380. * pc->mem_cgroup is not cleared here. It will be accessed when it's
  2381. * freed from LRU. This is safe because uncharged page is expected not
  2382. * to be reused (freed soon). Exception is SwapCache, it's handled by
  2383. * special functions.
  2384. */
  2385. unlock_page_cgroup(pc);
  2386. /*
  2387. * even after unlock, we have mem->res.usage here and this memcg
  2388. * will never be freed.
  2389. */
  2390. memcg_check_events(mem, page);
  2391. if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) {
  2392. mem_cgroup_swap_statistics(mem, true);
  2393. mem_cgroup_get(mem);
  2394. }
  2395. if (!mem_cgroup_is_root(mem))
  2396. __do_uncharge(mem, ctype, page_size);
  2397. return mem;
  2398. unlock_out:
  2399. unlock_page_cgroup(pc);
  2400. return NULL;
  2401. }
  2402. void mem_cgroup_uncharge_page(struct page *page)
  2403. {
  2404. /* early check. */
  2405. if (page_mapped(page))
  2406. return;
  2407. if (page->mapping && !PageAnon(page))
  2408. return;
  2409. __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED);
  2410. }
  2411. void mem_cgroup_uncharge_cache_page(struct page *page)
  2412. {
  2413. VM_BUG_ON(page_mapped(page));
  2414. VM_BUG_ON(page->mapping);
  2415. __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE);
  2416. }
  2417. /*
  2418. * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate.
  2419. * In that cases, pages are freed continuously and we can expect pages
  2420. * are in the same memcg. All these calls itself limits the number of
  2421. * pages freed at once, then uncharge_start/end() is called properly.
  2422. * This may be called prural(2) times in a context,
  2423. */
  2424. void mem_cgroup_uncharge_start(void)
  2425. {
  2426. current->memcg_batch.do_batch++;
  2427. /* We can do nest. */
  2428. if (current->memcg_batch.do_batch == 1) {
  2429. current->memcg_batch.memcg = NULL;
  2430. current->memcg_batch.bytes = 0;
  2431. current->memcg_batch.memsw_bytes = 0;
  2432. }
  2433. }
  2434. void mem_cgroup_uncharge_end(void)
  2435. {
  2436. struct memcg_batch_info *batch = &current->memcg_batch;
  2437. if (!batch->do_batch)
  2438. return;
  2439. batch->do_batch--;
  2440. if (batch->do_batch) /* If stacked, do nothing. */
  2441. return;
  2442. if (!batch->memcg)
  2443. return;
  2444. /*
  2445. * This "batch->memcg" is valid without any css_get/put etc...
  2446. * bacause we hide charges behind us.
  2447. */
  2448. if (batch->bytes)
  2449. res_counter_uncharge(&batch->memcg->res, batch->bytes);
  2450. if (batch->memsw_bytes)
  2451. res_counter_uncharge(&batch->memcg->memsw, batch->memsw_bytes);
  2452. memcg_oom_recover(batch->memcg);
  2453. /* forget this pointer (for sanity check) */
  2454. batch->memcg = NULL;
  2455. }
  2456. #ifdef CONFIG_SWAP
  2457. /*
  2458. * called after __delete_from_swap_cache() and drop "page" account.
  2459. * memcg information is recorded to swap_cgroup of "ent"
  2460. */
  2461. void
  2462. mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
  2463. {
  2464. struct mem_cgroup *memcg;
  2465. int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT;
  2466. if (!swapout) /* this was a swap cache but the swap is unused ! */
  2467. ctype = MEM_CGROUP_CHARGE_TYPE_DROP;
  2468. memcg = __mem_cgroup_uncharge_common(page, ctype);
  2469. /*
  2470. * record memcg information, if swapout && memcg != NULL,
  2471. * mem_cgroup_get() was called in uncharge().
  2472. */
  2473. if (do_swap_account && swapout && memcg)
  2474. swap_cgroup_record(ent, css_id(&memcg->css));
  2475. }
  2476. #endif
  2477. #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
  2478. /*
  2479. * called from swap_entry_free(). remove record in swap_cgroup and
  2480. * uncharge "memsw" account.
  2481. */
  2482. void mem_cgroup_uncharge_swap(swp_entry_t ent)
  2483. {
  2484. struct mem_cgroup *memcg;
  2485. unsigned short id;
  2486. if (!do_swap_account)
  2487. return;
  2488. id = swap_cgroup_record(ent, 0);
  2489. rcu_read_lock();
  2490. memcg = mem_cgroup_lookup(id);
  2491. if (memcg) {
  2492. /*
  2493. * We uncharge this because swap is freed.
  2494. * This memcg can be obsolete one. We avoid calling css_tryget
  2495. */
  2496. if (!mem_cgroup_is_root(memcg))
  2497. res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
  2498. mem_cgroup_swap_statistics(memcg, false);
  2499. mem_cgroup_put(memcg);
  2500. }
  2501. rcu_read_unlock();
  2502. }
  2503. /**
  2504. * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
  2505. * @entry: swap entry to be moved
  2506. * @from: mem_cgroup which the entry is moved from
  2507. * @to: mem_cgroup which the entry is moved to
  2508. * @need_fixup: whether we should fixup res_counters and refcounts.
  2509. *
  2510. * It succeeds only when the swap_cgroup's record for this entry is the same
  2511. * as the mem_cgroup's id of @from.
  2512. *
  2513. * Returns 0 on success, -EINVAL on failure.
  2514. *
  2515. * The caller must have charged to @to, IOW, called res_counter_charge() about
  2516. * both res and memsw, and called css_get().
  2517. */
  2518. static int mem_cgroup_move_swap_account(swp_entry_t entry,
  2519. struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup)
  2520. {
  2521. unsigned short old_id, new_id;
  2522. old_id = css_id(&from->css);
  2523. new_id = css_id(&to->css);
  2524. if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
  2525. mem_cgroup_swap_statistics(from, false);
  2526. mem_cgroup_swap_statistics(to, true);
  2527. /*
  2528. * This function is only called from task migration context now.
  2529. * It postpones res_counter and refcount handling till the end
  2530. * of task migration(mem_cgroup_clear_mc()) for performance
  2531. * improvement. But we cannot postpone mem_cgroup_get(to)
  2532. * because if the process that has been moved to @to does
  2533. * swap-in, the refcount of @to might be decreased to 0.
  2534. */
  2535. mem_cgroup_get(to);
  2536. if (need_fixup) {
  2537. if (!mem_cgroup_is_root(from))
  2538. res_counter_uncharge(&from->memsw, PAGE_SIZE);
  2539. mem_cgroup_put(from);
  2540. /*
  2541. * we charged both to->res and to->memsw, so we should
  2542. * uncharge to->res.
  2543. */
  2544. if (!mem_cgroup_is_root(to))
  2545. res_counter_uncharge(&to->res, PAGE_SIZE);
  2546. }
  2547. return 0;
  2548. }
  2549. return -EINVAL;
  2550. }
  2551. #else
  2552. static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
  2553. struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup)
  2554. {
  2555. return -EINVAL;
  2556. }
  2557. #endif
  2558. /*
  2559. * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
  2560. * page belongs to.
  2561. */
  2562. int mem_cgroup_prepare_migration(struct page *page,
  2563. struct page *newpage, struct mem_cgroup **ptr, gfp_t gfp_mask)
  2564. {
  2565. struct page_cgroup *pc;
  2566. struct mem_cgroup *mem = NULL;
  2567. enum charge_type ctype;
  2568. int ret = 0;
  2569. VM_BUG_ON(PageTransHuge(page));
  2570. if (mem_cgroup_disabled())
  2571. return 0;
  2572. pc = lookup_page_cgroup(page);
  2573. lock_page_cgroup(pc);
  2574. if (PageCgroupUsed(pc)) {
  2575. mem = pc->mem_cgroup;
  2576. css_get(&mem->css);
  2577. /*
  2578. * At migrating an anonymous page, its mapcount goes down
  2579. * to 0 and uncharge() will be called. But, even if it's fully
  2580. * unmapped, migration may fail and this page has to be
  2581. * charged again. We set MIGRATION flag here and delay uncharge
  2582. * until end_migration() is called
  2583. *
  2584. * Corner Case Thinking
  2585. * A)
  2586. * When the old page was mapped as Anon and it's unmap-and-freed
  2587. * while migration was ongoing.
  2588. * If unmap finds the old page, uncharge() of it will be delayed
  2589. * until end_migration(). If unmap finds a new page, it's
  2590. * uncharged when it make mapcount to be 1->0. If unmap code
  2591. * finds swap_migration_entry, the new page will not be mapped
  2592. * and end_migration() will find it(mapcount==0).
  2593. *
  2594. * B)
  2595. * When the old page was mapped but migraion fails, the kernel
  2596. * remaps it. A charge for it is kept by MIGRATION flag even
  2597. * if mapcount goes down to 0. We can do remap successfully
  2598. * without charging it again.
  2599. *
  2600. * C)
  2601. * The "old" page is under lock_page() until the end of
  2602. * migration, so, the old page itself will not be swapped-out.
  2603. * If the new page is swapped out before end_migraton, our
  2604. * hook to usual swap-out path will catch the event.
  2605. */
  2606. if (PageAnon(page))
  2607. SetPageCgroupMigration(pc);
  2608. }
  2609. unlock_page_cgroup(pc);
  2610. /*
  2611. * If the page is not charged at this point,
  2612. * we return here.
  2613. */
  2614. if (!mem)
  2615. return 0;
  2616. *ptr = mem;
  2617. ret = __mem_cgroup_try_charge(NULL, gfp_mask, ptr, false, PAGE_SIZE);
  2618. css_put(&mem->css);/* drop extra refcnt */
  2619. if (ret || *ptr == NULL) {
  2620. if (PageAnon(page)) {
  2621. lock_page_cgroup(pc);
  2622. ClearPageCgroupMigration(pc);
  2623. unlock_page_cgroup(pc);
  2624. /*
  2625. * The old page may be fully unmapped while we kept it.
  2626. */
  2627. mem_cgroup_uncharge_page(page);
  2628. }
  2629. return -ENOMEM;
  2630. }
  2631. /*
  2632. * We charge new page before it's used/mapped. So, even if unlock_page()
  2633. * is called before end_migration, we can catch all events on this new
  2634. * page. In the case new page is migrated but not remapped, new page's
  2635. * mapcount will be finally 0 and we call uncharge in end_migration().
  2636. */
  2637. pc = lookup_page_cgroup(newpage);
  2638. if (PageAnon(page))
  2639. ctype = MEM_CGROUP_CHARGE_TYPE_MAPPED;
  2640. else if (page_is_file_cache(page))
  2641. ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
  2642. else
  2643. ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM;
  2644. __mem_cgroup_commit_charge(mem, pc, ctype, PAGE_SIZE);
  2645. return ret;
  2646. }
  2647. /* remove redundant charge if migration failed*/
  2648. void mem_cgroup_end_migration(struct mem_cgroup *mem,
  2649. struct page *oldpage, struct page *newpage, bool migration_ok)
  2650. {
  2651. struct page *used, *unused;
  2652. struct page_cgroup *pc;
  2653. if (!mem)
  2654. return;
  2655. /* blocks rmdir() */
  2656. cgroup_exclude_rmdir(&mem->css);
  2657. if (!migration_ok) {
  2658. used = oldpage;
  2659. unused = newpage;
  2660. } else {
  2661. used = newpage;
  2662. unused = oldpage;
  2663. }
  2664. /*
  2665. * We disallowed uncharge of pages under migration because mapcount
  2666. * of the page goes down to zero, temporarly.
  2667. * Clear the flag and check the page should be charged.
  2668. */
  2669. pc = lookup_page_cgroup(oldpage);
  2670. lock_page_cgroup(pc);
  2671. ClearPageCgroupMigration(pc);
  2672. unlock_page_cgroup(pc);
  2673. __mem_cgroup_uncharge_common(unused, MEM_CGROUP_CHARGE_TYPE_FORCE);
  2674. /*
  2675. * If a page is a file cache, radix-tree replacement is very atomic
  2676. * and we can skip this check. When it was an Anon page, its mapcount
  2677. * goes down to 0. But because we added MIGRATION flage, it's not
  2678. * uncharged yet. There are several case but page->mapcount check
  2679. * and USED bit check in mem_cgroup_uncharge_page() will do enough
  2680. * check. (see prepare_charge() also)
  2681. */
  2682. if (PageAnon(used))
  2683. mem_cgroup_uncharge_page(used);
  2684. /*
  2685. * At migration, we may charge account against cgroup which has no
  2686. * tasks.
  2687. * So, rmdir()->pre_destroy() can be called while we do this charge.
  2688. * In that case, we need to call pre_destroy() again. check it here.
  2689. */
  2690. cgroup_release_and_wakeup_rmdir(&mem->css);
  2691. }
  2692. /*
  2693. * A call to try to shrink memory usage on charge failure at shmem's swapin.
  2694. * Calling hierarchical_reclaim is not enough because we should update
  2695. * last_oom_jiffies to prevent pagefault_out_of_memory from invoking global OOM.
  2696. * Moreover considering hierarchy, we should reclaim from the mem_over_limit,
  2697. * not from the memcg which this page would be charged to.
  2698. * try_charge_swapin does all of these works properly.
  2699. */
  2700. int mem_cgroup_shmem_charge_fallback(struct page *page,
  2701. struct mm_struct *mm,
  2702. gfp_t gfp_mask)
  2703. {
  2704. struct mem_cgroup *mem = NULL;
  2705. int ret;
  2706. if (mem_cgroup_disabled())
  2707. return 0;
  2708. ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
  2709. if (!ret)
  2710. mem_cgroup_cancel_charge_swapin(mem); /* it does !mem check */
  2711. return ret;
  2712. }
  2713. static DEFINE_MUTEX(set_limit_mutex);
  2714. static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
  2715. unsigned long long val)
  2716. {
  2717. int retry_count;
  2718. u64 memswlimit, memlimit;
  2719. int ret = 0;
  2720. int children = mem_cgroup_count_children(memcg);
  2721. u64 curusage, oldusage;
  2722. int enlarge;
  2723. /*
  2724. * For keeping hierarchical_reclaim simple, how long we should retry
  2725. * is depends on callers. We set our retry-count to be function
  2726. * of # of children which we should visit in this loop.
  2727. */
  2728. retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;
  2729. oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
  2730. enlarge = 0;
  2731. while (retry_count) {
  2732. if (signal_pending(current)) {
  2733. ret = -EINTR;
  2734. break;
  2735. }
  2736. /*
  2737. * Rather than hide all in some function, I do this in
  2738. * open coded manner. You see what this really does.
  2739. * We have to guarantee mem->res.limit < mem->memsw.limit.
  2740. */
  2741. mutex_lock(&set_limit_mutex);
  2742. memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
  2743. if (memswlimit < val) {
  2744. ret = -EINVAL;
  2745. mutex_unlock(&set_limit_mutex);
  2746. break;
  2747. }
  2748. memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
  2749. if (memlimit < val)
  2750. enlarge = 1;
  2751. ret = res_counter_set_limit(&memcg->res, val);
  2752. if (!ret) {
  2753. if (memswlimit == val)
  2754. memcg->memsw_is_minimum = true;
  2755. else
  2756. memcg->memsw_is_minimum = false;
  2757. }
  2758. mutex_unlock(&set_limit_mutex);
  2759. if (!ret)
  2760. break;
  2761. mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
  2762. MEM_CGROUP_RECLAIM_SHRINK);
  2763. curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
  2764. /* Usage is reduced ? */
  2765. if (curusage >= oldusage)
  2766. retry_count--;
  2767. else
  2768. oldusage = curusage;
  2769. }
  2770. if (!ret && enlarge)
  2771. memcg_oom_recover(memcg);
  2772. return ret;
  2773. }
  2774. static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
  2775. unsigned long long val)
  2776. {
  2777. int retry_count;
  2778. u64 memlimit, memswlimit, oldusage, curusage;
  2779. int children = mem_cgroup_count_children(memcg);
  2780. int ret = -EBUSY;
  2781. int enlarge = 0;
  2782. /* see mem_cgroup_resize_res_limit */
  2783. retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
  2784. oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
  2785. while (retry_count) {
  2786. if (signal_pending(current)) {
  2787. ret = -EINTR;
  2788. break;
  2789. }
  2790. /*
  2791. * Rather than hide all in some function, I do this in
  2792. * open coded manner. You see what this really does.
  2793. * We have to guarantee mem->res.limit < mem->memsw.limit.
  2794. */
  2795. mutex_lock(&set_limit_mutex);
  2796. memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
  2797. if (memlimit > val) {
  2798. ret = -EINVAL;
  2799. mutex_unlock(&set_limit_mutex);
  2800. break;
  2801. }
  2802. memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
  2803. if (memswlimit < val)
  2804. enlarge = 1;
  2805. ret = res_counter_set_limit(&memcg->memsw, val);
  2806. if (!ret) {
  2807. if (memlimit == val)
  2808. memcg->memsw_is_minimum = true;
  2809. else
  2810. memcg->memsw_is_minimum = false;
  2811. }
  2812. mutex_unlock(&set_limit_mutex);
  2813. if (!ret)
  2814. break;
  2815. mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
  2816. MEM_CGROUP_RECLAIM_NOSWAP |
  2817. MEM_CGROUP_RECLAIM_SHRINK);
  2818. curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
  2819. /* Usage is reduced ? */
  2820. if (curusage >= oldusage)
  2821. retry_count--;
  2822. else
  2823. oldusage = curusage;
  2824. }
  2825. if (!ret && enlarge)
  2826. memcg_oom_recover(memcg);
  2827. return ret;
  2828. }
  2829. unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
  2830. gfp_t gfp_mask)
  2831. {
  2832. unsigned long nr_reclaimed = 0;
  2833. struct mem_cgroup_per_zone *mz, *next_mz = NULL;
  2834. unsigned long reclaimed;
  2835. int loop = 0;
  2836. struct mem_cgroup_tree_per_zone *mctz;
  2837. unsigned long long excess;
  2838. if (order > 0)
  2839. return 0;
  2840. mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone));
  2841. /*
  2842. * This loop can run a while, specially if mem_cgroup's continuously
  2843. * keep exceeding their soft limit and putting the system under
  2844. * pressure
  2845. */
  2846. do {
  2847. if (next_mz)
  2848. mz = next_mz;
  2849. else
  2850. mz = mem_cgroup_largest_soft_limit_node(mctz);
  2851. if (!mz)
  2852. break;
  2853. reclaimed = mem_cgroup_hierarchical_reclaim(mz->mem, zone,
  2854. gfp_mask,
  2855. MEM_CGROUP_RECLAIM_SOFT);
  2856. nr_reclaimed += reclaimed;
  2857. spin_lock(&mctz->lock);
  2858. /*
  2859. * If we failed to reclaim anything from this memory cgroup
  2860. * it is time to move on to the next cgroup
  2861. */
  2862. next_mz = NULL;
  2863. if (!reclaimed) {
  2864. do {
  2865. /*
  2866. * Loop until we find yet another one.
  2867. *
  2868. * By the time we get the soft_limit lock
  2869. * again, someone might have aded the
  2870. * group back on the RB tree. Iterate to
  2871. * make sure we get a different mem.
  2872. * mem_cgroup_largest_soft_limit_node returns
  2873. * NULL if no other cgroup is present on
  2874. * the tree
  2875. */
  2876. next_mz =
  2877. __mem_cgroup_largest_soft_limit_node(mctz);
  2878. if (next_mz == mz) {
  2879. css_put(&next_mz->mem->css);
  2880. next_mz = NULL;
  2881. } else /* next_mz == NULL or other memcg */
  2882. break;
  2883. } while (1);
  2884. }
  2885. __mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
  2886. excess = res_counter_soft_limit_excess(&mz->mem->res);
  2887. /*
  2888. * One school of thought says that we should not add
  2889. * back the node to the tree if reclaim returns 0.
  2890. * But our reclaim could return 0, simply because due
  2891. * to priority we are exposing a smaller subset of
  2892. * memory to reclaim from. Consider this as a longer
  2893. * term TODO.
  2894. */
  2895. /* If excess == 0, no tree ops */
  2896. __mem_cgroup_insert_exceeded(mz->mem, mz, mctz, excess);
  2897. spin_unlock(&mctz->lock);
  2898. css_put(&mz->mem->css);
  2899. loop++;
  2900. /*
  2901. * Could not reclaim anything and there are no more
  2902. * mem cgroups to try or we seem to be looping without
  2903. * reclaiming anything.
  2904. */
  2905. if (!nr_reclaimed &&
  2906. (next_mz == NULL ||
  2907. loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
  2908. break;
  2909. } while (!nr_reclaimed);
  2910. if (next_mz)
  2911. css_put(&next_mz->mem->css);
  2912. return nr_reclaimed;
  2913. }
  2914. /*
  2915. * This routine traverse page_cgroup in given list and drop them all.
  2916. * *And* this routine doesn't reclaim page itself, just removes page_cgroup.
  2917. */
  2918. static int mem_cgroup_force_empty_list(struct mem_cgroup *mem,
  2919. int node, int zid, enum lru_list lru)
  2920. {
  2921. struct zone *zone;
  2922. struct mem_cgroup_per_zone *mz;
  2923. struct page_cgroup *pc, *busy;
  2924. unsigned long flags, loop;
  2925. struct list_head *list;
  2926. int ret = 0;
  2927. zone = &NODE_DATA(node)->node_zones[zid];
  2928. mz = mem_cgroup_zoneinfo(mem, node, zid);
  2929. list = &mz->lists[lru];
  2930. loop = MEM_CGROUP_ZSTAT(mz, lru);
  2931. /* give some margin against EBUSY etc...*/
  2932. loop += 256;
  2933. busy = NULL;
  2934. while (loop--) {
  2935. ret = 0;
  2936. spin_lock_irqsave(&zone->lru_lock, flags);
  2937. if (list_empty(list)) {
  2938. spin_unlock_irqrestore(&zone->lru_lock, flags);
  2939. break;
  2940. }
  2941. pc = list_entry(list->prev, struct page_cgroup, lru);
  2942. if (busy == pc) {
  2943. list_move(&pc->lru, list);
  2944. busy = NULL;
  2945. spin_unlock_irqrestore(&zone->lru_lock, flags);
  2946. continue;
  2947. }
  2948. spin_unlock_irqrestore(&zone->lru_lock, flags);
  2949. ret = mem_cgroup_move_parent(pc, mem, GFP_KERNEL);
  2950. if (ret == -ENOMEM)
  2951. break;
  2952. if (ret == -EBUSY || ret == -EINVAL) {
  2953. /* found lock contention or "pc" is obsolete. */
  2954. busy = pc;
  2955. cond_resched();
  2956. } else
  2957. busy = NULL;
  2958. }
  2959. if (!ret && !list_empty(list))
  2960. return -EBUSY;
  2961. return ret;
  2962. }
  2963. /*
  2964. * make mem_cgroup's charge to be 0 if there is no task.
  2965. * This enables deleting this mem_cgroup.
  2966. */
  2967. static int mem_cgroup_force_empty(struct mem_cgroup *mem, bool free_all)
  2968. {
  2969. int ret;
  2970. int node, zid, shrink;
  2971. int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
  2972. struct cgroup *cgrp = mem->css.cgroup;
  2973. css_get(&mem->css);
  2974. shrink = 0;
  2975. /* should free all ? */
  2976. if (free_all)
  2977. goto try_to_free;
  2978. move_account:
  2979. do {
  2980. ret = -EBUSY;
  2981. if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
  2982. goto out;
  2983. ret = -EINTR;
  2984. if (signal_pending(current))
  2985. goto out;
  2986. /* This is for making all *used* pages to be on LRU. */
  2987. lru_add_drain_all();
  2988. drain_all_stock_sync();
  2989. ret = 0;
  2990. mem_cgroup_start_move(mem);
  2991. for_each_node_state(node, N_HIGH_MEMORY) {
  2992. for (zid = 0; !ret && zid < MAX_NR_ZONES; zid++) {
  2993. enum lru_list l;
  2994. for_each_lru(l) {
  2995. ret = mem_cgroup_force_empty_list(mem,
  2996. node, zid, l);
  2997. if (ret)
  2998. break;
  2999. }
  3000. }
  3001. if (ret)
  3002. break;
  3003. }
  3004. mem_cgroup_end_move(mem);
  3005. memcg_oom_recover(mem);
  3006. /* it seems parent cgroup doesn't have enough mem */
  3007. if (ret == -ENOMEM)
  3008. goto try_to_free;
  3009. cond_resched();
  3010. /* "ret" should also be checked to ensure all lists are empty. */
  3011. } while (mem->res.usage > 0 || ret);
  3012. out:
  3013. css_put(&mem->css);
  3014. return ret;
  3015. try_to_free:
  3016. /* returns EBUSY if there is a task or if we come here twice. */
  3017. if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children) || shrink) {
  3018. ret = -EBUSY;
  3019. goto out;
  3020. }
  3021. /* we call try-to-free pages for make this cgroup empty */
  3022. lru_add_drain_all();
  3023. /* try to free all pages in this cgroup */
  3024. shrink = 1;
  3025. while (nr_retries && mem->res.usage > 0) {
  3026. int progress;
  3027. if (signal_pending(current)) {
  3028. ret = -EINTR;
  3029. goto out;
  3030. }
  3031. progress = try_to_free_mem_cgroup_pages(mem, GFP_KERNEL,
  3032. false, get_swappiness(mem));
  3033. if (!progress) {
  3034. nr_retries--;
  3035. /* maybe some writeback is necessary */
  3036. congestion_wait(BLK_RW_ASYNC, HZ/10);
  3037. }
  3038. }
  3039. lru_add_drain();
  3040. /* try move_account...there may be some *locked* pages. */
  3041. goto move_account;
  3042. }
  3043. int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event)
  3044. {
  3045. return mem_cgroup_force_empty(mem_cgroup_from_cont(cont), true);
  3046. }
  3047. static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft)
  3048. {
  3049. return mem_cgroup_from_cont(cont)->use_hierarchy;
  3050. }
  3051. static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
  3052. u64 val)
  3053. {
  3054. int retval = 0;
  3055. struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
  3056. struct cgroup *parent = cont->parent;
  3057. struct mem_cgroup *parent_mem = NULL;
  3058. if (parent)
  3059. parent_mem = mem_cgroup_from_cont(parent);
  3060. cgroup_lock();
  3061. /*
  3062. * If parent's use_hierarchy is set, we can't make any modifications
  3063. * in the child subtrees. If it is unset, then the change can
  3064. * occur, provided the current cgroup has no children.
  3065. *
  3066. * For the root cgroup, parent_mem is NULL, we allow value to be
  3067. * set if there are no children.
  3068. */
  3069. if ((!parent_mem || !parent_mem->use_hierarchy) &&
  3070. (val == 1 || val == 0)) {
  3071. if (list_empty(&cont->children))
  3072. mem->use_hierarchy = val;
  3073. else
  3074. retval = -EBUSY;
  3075. } else
  3076. retval = -EINVAL;
  3077. cgroup_unlock();
  3078. return retval;
  3079. }
  3080. static u64 mem_cgroup_get_recursive_idx_stat(struct mem_cgroup *mem,
  3081. enum mem_cgroup_stat_index idx)
  3082. {
  3083. struct mem_cgroup *iter;
  3084. s64 val = 0;
  3085. /* each per cpu's value can be minus.Then, use s64 */
  3086. for_each_mem_cgroup_tree(iter, mem)
  3087. val += mem_cgroup_read_stat(iter, idx);
  3088. if (val < 0) /* race ? */
  3089. val = 0;
  3090. return val;
  3091. }
  3092. static inline u64 mem_cgroup_usage(struct mem_cgroup *mem, bool swap)
  3093. {
  3094. u64 val;
  3095. if (!mem_cgroup_is_root(mem)) {
  3096. if (!swap)
  3097. return res_counter_read_u64(&mem->res, RES_USAGE);
  3098. else
  3099. return res_counter_read_u64(&mem->memsw, RES_USAGE);
  3100. }
  3101. val = mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_CACHE);
  3102. val += mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_RSS);
  3103. if (swap)
  3104. val += mem_cgroup_get_recursive_idx_stat(mem,
  3105. MEM_CGROUP_STAT_SWAPOUT);
  3106. return val << PAGE_SHIFT;
  3107. }
  3108. static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft)
  3109. {
  3110. struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
  3111. u64 val;
  3112. int type, name;
  3113. type = MEMFILE_TYPE(cft->private);
  3114. name = MEMFILE_ATTR(cft->private);
  3115. switch (type) {
  3116. case _MEM:
  3117. if (name == RES_USAGE)
  3118. val = mem_cgroup_usage(mem, false);
  3119. else
  3120. val = res_counter_read_u64(&mem->res, name);
  3121. break;
  3122. case _MEMSWAP:
  3123. if (name == RES_USAGE)
  3124. val = mem_cgroup_usage(mem, true);
  3125. else
  3126. val = res_counter_read_u64(&mem->memsw, name);
  3127. break;
  3128. default:
  3129. BUG();
  3130. break;
  3131. }
  3132. return val;
  3133. }
  3134. /*
  3135. * The user of this function is...
  3136. * RES_LIMIT.
  3137. */
  3138. static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
  3139. const char *buffer)
  3140. {
  3141. struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
  3142. int type, name;
  3143. unsigned long long val;
  3144. int ret;
  3145. type = MEMFILE_TYPE(cft->private);
  3146. name = MEMFILE_ATTR(cft->private);
  3147. switch (name) {
  3148. case RES_LIMIT:
  3149. if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
  3150. ret = -EINVAL;
  3151. break;
  3152. }
  3153. /* This function does all necessary parse...reuse it */
  3154. ret = res_counter_memparse_write_strategy(buffer, &val);
  3155. if (ret)
  3156. break;
  3157. if (type == _MEM)
  3158. ret = mem_cgroup_resize_limit(memcg, val);
  3159. else
  3160. ret = mem_cgroup_resize_memsw_limit(memcg, val);
  3161. break;
  3162. case RES_SOFT_LIMIT:
  3163. ret = res_counter_memparse_write_strategy(buffer, &val);
  3164. if (ret)
  3165. break;
  3166. /*
  3167. * For memsw, soft limits are hard to implement in terms
  3168. * of semantics, for now, we support soft limits for
  3169. * control without swap
  3170. */
  3171. if (type == _MEM)
  3172. ret = res_counter_set_soft_limit(&memcg->res, val);
  3173. else
  3174. ret = -EINVAL;
  3175. break;
  3176. default:
  3177. ret = -EINVAL; /* should be BUG() ? */
  3178. break;
  3179. }
  3180. return ret;
  3181. }
  3182. static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg,
  3183. unsigned long long *mem_limit, unsigned long long *memsw_limit)
  3184. {
  3185. struct cgroup *cgroup;
  3186. unsigned long long min_limit, min_memsw_limit, tmp;
  3187. min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
  3188. min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
  3189. cgroup = memcg->css.cgroup;
  3190. if (!memcg->use_hierarchy)
  3191. goto out;
  3192. while (cgroup->parent) {
  3193. cgroup = cgroup->parent;
  3194. memcg = mem_cgroup_from_cont(cgroup);
  3195. if (!memcg->use_hierarchy)
  3196. break;
  3197. tmp = res_counter_read_u64(&memcg->res, RES_LIMIT);
  3198. min_limit = min(min_limit, tmp);
  3199. tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
  3200. min_memsw_limit = min(min_memsw_limit, tmp);
  3201. }
  3202. out:
  3203. *mem_limit = min_limit;
  3204. *memsw_limit = min_memsw_limit;
  3205. return;
  3206. }
  3207. static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
  3208. {
  3209. struct mem_cgroup *mem;
  3210. int type, name;
  3211. mem = mem_cgroup_from_cont(cont);
  3212. type = MEMFILE_TYPE(event);
  3213. name = MEMFILE_ATTR(event);
  3214. switch (name) {
  3215. case RES_MAX_USAGE:
  3216. if (type == _MEM)
  3217. res_counter_reset_max(&mem->res);
  3218. else
  3219. res_counter_reset_max(&mem->memsw);
  3220. break;
  3221. case RES_FAILCNT:
  3222. if (type == _MEM)
  3223. res_counter_reset_failcnt(&mem->res);
  3224. else
  3225. res_counter_reset_failcnt(&mem->memsw);
  3226. break;
  3227. }
  3228. return 0;
  3229. }
  3230. static u64 mem_cgroup_move_charge_read(struct cgroup *cgrp,
  3231. struct cftype *cft)
  3232. {
  3233. return mem_cgroup_from_cont(cgrp)->move_charge_at_immigrate;
  3234. }
  3235. #ifdef CONFIG_MMU
  3236. static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
  3237. struct cftype *cft, u64 val)
  3238. {
  3239. struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
  3240. if (val >= (1 << NR_MOVE_TYPE))
  3241. return -EINVAL;
  3242. /*
  3243. * We check this value several times in both in can_attach() and
  3244. * attach(), so we need cgroup lock to prevent this value from being
  3245. * inconsistent.
  3246. */
  3247. cgroup_lock();
  3248. mem->move_charge_at_immigrate = val;
  3249. cgroup_unlock();
  3250. return 0;
  3251. }
  3252. #else
  3253. static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
  3254. struct cftype *cft, u64 val)
  3255. {
  3256. return -ENOSYS;
  3257. }
  3258. #endif
  3259. /* For read statistics */
  3260. enum {
  3261. MCS_CACHE,
  3262. MCS_RSS,
  3263. MCS_FILE_MAPPED,
  3264. MCS_PGPGIN,
  3265. MCS_PGPGOUT,
  3266. MCS_SWAP,
  3267. MCS_INACTIVE_ANON,
  3268. MCS_ACTIVE_ANON,
  3269. MCS_INACTIVE_FILE,
  3270. MCS_ACTIVE_FILE,
  3271. MCS_UNEVICTABLE,
  3272. NR_MCS_STAT,
  3273. };
  3274. struct mcs_total_stat {
  3275. s64 stat[NR_MCS_STAT];
  3276. };
  3277. struct {
  3278. char *local_name;
  3279. char *total_name;
  3280. } memcg_stat_strings[NR_MCS_STAT] = {
  3281. {"cache", "total_cache"},
  3282. {"rss", "total_rss"},
  3283. {"mapped_file", "total_mapped_file"},
  3284. {"pgpgin", "total_pgpgin"},
  3285. {"pgpgout", "total_pgpgout"},
  3286. {"swap", "total_swap"},
  3287. {"inactive_anon", "total_inactive_anon"},
  3288. {"active_anon", "total_active_anon"},
  3289. {"inactive_file", "total_inactive_file"},
  3290. {"active_file", "total_active_file"},
  3291. {"unevictable", "total_unevictable"}
  3292. };
  3293. static void
  3294. mem_cgroup_get_local_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
  3295. {
  3296. s64 val;
  3297. /* per cpu stat */
  3298. val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE);
  3299. s->stat[MCS_CACHE] += val * PAGE_SIZE;
  3300. val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS);
  3301. s->stat[MCS_RSS] += val * PAGE_SIZE;
  3302. val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_FILE_MAPPED);
  3303. s->stat[MCS_FILE_MAPPED] += val * PAGE_SIZE;
  3304. val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGIN_COUNT);
  3305. s->stat[MCS_PGPGIN] += val;
  3306. val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGOUT_COUNT);
  3307. s->stat[MCS_PGPGOUT] += val;
  3308. if (do_swap_account) {
  3309. val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_SWAPOUT);
  3310. s->stat[MCS_SWAP] += val * PAGE_SIZE;
  3311. }
  3312. /* per zone stat */
  3313. val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_ANON);
  3314. s->stat[MCS_INACTIVE_ANON] += val * PAGE_SIZE;
  3315. val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_ANON);
  3316. s->stat[MCS_ACTIVE_ANON] += val * PAGE_SIZE;
  3317. val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_FILE);
  3318. s->stat[MCS_INACTIVE_FILE] += val * PAGE_SIZE;
  3319. val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_FILE);
  3320. s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE;
  3321. val = mem_cgroup_get_local_zonestat(mem, LRU_UNEVICTABLE);
  3322. s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE;
  3323. }
  3324. static void
  3325. mem_cgroup_get_total_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
  3326. {
  3327. struct mem_cgroup *iter;
  3328. for_each_mem_cgroup_tree(iter, mem)
  3329. mem_cgroup_get_local_stat(iter, s);
  3330. }
  3331. static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
  3332. struct cgroup_map_cb *cb)
  3333. {
  3334. struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
  3335. struct mcs_total_stat mystat;
  3336. int i;
  3337. memset(&mystat, 0, sizeof(mystat));
  3338. mem_cgroup_get_local_stat(mem_cont, &mystat);
  3339. for (i = 0; i < NR_MCS_STAT; i++) {
  3340. if (i == MCS_SWAP && !do_swap_account)
  3341. continue;
  3342. cb->fill(cb, memcg_stat_strings[i].local_name, mystat.stat[i]);
  3343. }
  3344. /* Hierarchical information */
  3345. {
  3346. unsigned long long limit, memsw_limit;
  3347. memcg_get_hierarchical_limit(mem_cont, &limit, &memsw_limit);
  3348. cb->fill(cb, "hierarchical_memory_limit", limit);
  3349. if (do_swap_account)
  3350. cb->fill(cb, "hierarchical_memsw_limit", memsw_limit);
  3351. }
  3352. memset(&mystat, 0, sizeof(mystat));
  3353. mem_cgroup_get_total_stat(mem_cont, &mystat);
  3354. for (i = 0; i < NR_MCS_STAT; i++) {
  3355. if (i == MCS_SWAP && !do_swap_account)
  3356. continue;
  3357. cb->fill(cb, memcg_stat_strings[i].total_name, mystat.stat[i]);
  3358. }
  3359. #ifdef CONFIG_DEBUG_VM
  3360. cb->fill(cb, "inactive_ratio", calc_inactive_ratio(mem_cont, NULL));
  3361. {
  3362. int nid, zid;
  3363. struct mem_cgroup_per_zone *mz;
  3364. unsigned long recent_rotated[2] = {0, 0};
  3365. unsigned long recent_scanned[2] = {0, 0};
  3366. for_each_online_node(nid)
  3367. for (zid = 0; zid < MAX_NR_ZONES; zid++) {
  3368. mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
  3369. recent_rotated[0] +=
  3370. mz->reclaim_stat.recent_rotated[0];
  3371. recent_rotated[1] +=
  3372. mz->reclaim_stat.recent_rotated[1];
  3373. recent_scanned[0] +=
  3374. mz->reclaim_stat.recent_scanned[0];
  3375. recent_scanned[1] +=
  3376. mz->reclaim_stat.recent_scanned[1];
  3377. }
  3378. cb->fill(cb, "recent_rotated_anon", recent_rotated[0]);
  3379. cb->fill(cb, "recent_rotated_file", recent_rotated[1]);
  3380. cb->fill(cb, "recent_scanned_anon", recent_scanned[0]);
  3381. cb->fill(cb, "recent_scanned_file", recent_scanned[1]);
  3382. }
  3383. #endif
  3384. return 0;
  3385. }
  3386. static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft)
  3387. {
  3388. struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
  3389. return get_swappiness(memcg);
  3390. }
  3391. static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft,
  3392. u64 val)
  3393. {
  3394. struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
  3395. struct mem_cgroup *parent;
  3396. if (val > 100)
  3397. return -EINVAL;
  3398. if (cgrp->parent == NULL)
  3399. return -EINVAL;
  3400. parent = mem_cgroup_from_cont(cgrp->parent);
  3401. cgroup_lock();
  3402. /* If under hierarchy, only empty-root can set this value */
  3403. if ((parent->use_hierarchy) ||
  3404. (memcg->use_hierarchy && !list_empty(&cgrp->children))) {
  3405. cgroup_unlock();
  3406. return -EINVAL;
  3407. }
  3408. spin_lock(&memcg->reclaim_param_lock);
  3409. memcg->swappiness = val;
  3410. spin_unlock(&memcg->reclaim_param_lock);
  3411. cgroup_unlock();
  3412. return 0;
  3413. }
  3414. static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
  3415. {
  3416. struct mem_cgroup_threshold_ary *t;
  3417. u64 usage;
  3418. int i;
  3419. rcu_read_lock();
  3420. if (!swap)
  3421. t = rcu_dereference(memcg->thresholds.primary);
  3422. else
  3423. t = rcu_dereference(memcg->memsw_thresholds.primary);
  3424. if (!t)
  3425. goto unlock;
  3426. usage = mem_cgroup_usage(memcg, swap);
  3427. /*
  3428. * current_threshold points to threshold just below usage.
  3429. * If it's not true, a threshold was crossed after last
  3430. * call of __mem_cgroup_threshold().
  3431. */
  3432. i = t->current_threshold;
  3433. /*
  3434. * Iterate backward over array of thresholds starting from
  3435. * current_threshold and check if a threshold is crossed.
  3436. * If none of thresholds below usage is crossed, we read
  3437. * only one element of the array here.
  3438. */
  3439. for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
  3440. eventfd_signal(t->entries[i].eventfd, 1);
  3441. /* i = current_threshold + 1 */
  3442. i++;
  3443. /*
  3444. * Iterate forward over array of thresholds starting from
  3445. * current_threshold+1 and check if a threshold is crossed.
  3446. * If none of thresholds above usage is crossed, we read
  3447. * only one element of the array here.
  3448. */
  3449. for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
  3450. eventfd_signal(t->entries[i].eventfd, 1);
  3451. /* Update current_threshold */
  3452. t->current_threshold = i - 1;
  3453. unlock:
  3454. rcu_read_unlock();
  3455. }
  3456. static void mem_cgroup_threshold(struct mem_cgroup *memcg)
  3457. {
  3458. while (memcg) {
  3459. __mem_cgroup_threshold(memcg, false);
  3460. if (do_swap_account)
  3461. __mem_cgroup_threshold(memcg, true);
  3462. memcg = parent_mem_cgroup(memcg);
  3463. }
  3464. }
  3465. static int compare_thresholds(const void *a, const void *b)
  3466. {
  3467. const struct mem_cgroup_threshold *_a = a;
  3468. const struct mem_cgroup_threshold *_b = b;
  3469. return _a->threshold - _b->threshold;
  3470. }
  3471. static int mem_cgroup_oom_notify_cb(struct mem_cgroup *mem)
  3472. {
  3473. struct mem_cgroup_eventfd_list *ev;
  3474. list_for_each_entry(ev, &mem->oom_notify, list)
  3475. eventfd_signal(ev->eventfd, 1);
  3476. return 0;
  3477. }
  3478. static void mem_cgroup_oom_notify(struct mem_cgroup *mem)
  3479. {
  3480. struct mem_cgroup *iter;
  3481. for_each_mem_cgroup_tree(iter, mem)
  3482. mem_cgroup_oom_notify_cb(iter);
  3483. }
  3484. static int mem_cgroup_usage_register_event(struct cgroup *cgrp,
  3485. struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
  3486. {
  3487. struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
  3488. struct mem_cgroup_thresholds *thresholds;
  3489. struct mem_cgroup_threshold_ary *new;
  3490. int type = MEMFILE_TYPE(cft->private);
  3491. u64 threshold, usage;
  3492. int i, size, ret;
  3493. ret = res_counter_memparse_write_strategy(args, &threshold);
  3494. if (ret)
  3495. return ret;
  3496. mutex_lock(&memcg->thresholds_lock);
  3497. if (type == _MEM)
  3498. thresholds = &memcg->thresholds;
  3499. else if (type == _MEMSWAP)
  3500. thresholds = &memcg->memsw_thresholds;
  3501. else
  3502. BUG();
  3503. usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
  3504. /* Check if a threshold crossed before adding a new one */
  3505. if (thresholds->primary)
  3506. __mem_cgroup_threshold(memcg, type == _MEMSWAP);
  3507. size = thresholds->primary ? thresholds->primary->size + 1 : 1;
  3508. /* Allocate memory for new array of thresholds */
  3509. new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
  3510. GFP_KERNEL);
  3511. if (!new) {
  3512. ret = -ENOMEM;
  3513. goto unlock;
  3514. }
  3515. new->size = size;
  3516. /* Copy thresholds (if any) to new array */
  3517. if (thresholds->primary) {
  3518. memcpy(new->entries, thresholds->primary->entries, (size - 1) *
  3519. sizeof(struct mem_cgroup_threshold));
  3520. }
  3521. /* Add new threshold */
  3522. new->entries[size - 1].eventfd = eventfd;
  3523. new->entries[size - 1].threshold = threshold;
  3524. /* Sort thresholds. Registering of new threshold isn't time-critical */
  3525. sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
  3526. compare_thresholds, NULL);
  3527. /* Find current threshold */
  3528. new->current_threshold = -1;
  3529. for (i = 0; i < size; i++) {
  3530. if (new->entries[i].threshold < usage) {
  3531. /*
  3532. * new->current_threshold will not be used until
  3533. * rcu_assign_pointer(), so it's safe to increment
  3534. * it here.
  3535. */
  3536. ++new->current_threshold;
  3537. }
  3538. }
  3539. /* Free old spare buffer and save old primary buffer as spare */
  3540. kfree(thresholds->spare);
  3541. thresholds->spare = thresholds->primary;
  3542. rcu_assign_pointer(thresholds->primary, new);
  3543. /* To be sure that nobody uses thresholds */
  3544. synchronize_rcu();
  3545. unlock:
  3546. mutex_unlock(&memcg->thresholds_lock);
  3547. return ret;
  3548. }
  3549. static void mem_cgroup_usage_unregister_event(struct cgroup *cgrp,
  3550. struct cftype *cft, struct eventfd_ctx *eventfd)
  3551. {
  3552. struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
  3553. struct mem_cgroup_thresholds *thresholds;
  3554. struct mem_cgroup_threshold_ary *new;
  3555. int type = MEMFILE_TYPE(cft->private);
  3556. u64 usage;
  3557. int i, j, size;
  3558. mutex_lock(&memcg->thresholds_lock);
  3559. if (type == _MEM)
  3560. thresholds = &memcg->thresholds;
  3561. else if (type == _MEMSWAP)
  3562. thresholds = &memcg->memsw_thresholds;
  3563. else
  3564. BUG();
  3565. /*
  3566. * Something went wrong if we trying to unregister a threshold
  3567. * if we don't have thresholds
  3568. */
  3569. BUG_ON(!thresholds);
  3570. usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
  3571. /* Check if a threshold crossed before removing */
  3572. __mem_cgroup_threshold(memcg, type == _MEMSWAP);
  3573. /* Calculate new number of threshold */
  3574. size = 0;
  3575. for (i = 0; i < thresholds->primary->size; i++) {
  3576. if (thresholds->primary->entries[i].eventfd != eventfd)
  3577. size++;
  3578. }
  3579. new = thresholds->spare;
  3580. /* Set thresholds array to NULL if we don't have thresholds */
  3581. if (!size) {
  3582. kfree(new);
  3583. new = NULL;
  3584. goto swap_buffers;
  3585. }
  3586. new->size = size;
  3587. /* Copy thresholds and find current threshold */
  3588. new->current_threshold = -1;
  3589. for (i = 0, j = 0; i < thresholds->primary->size; i++) {
  3590. if (thresholds->primary->entries[i].eventfd == eventfd)
  3591. continue;
  3592. new->entries[j] = thresholds->primary->entries[i];
  3593. if (new->entries[j].threshold < usage) {
  3594. /*
  3595. * new->current_threshold will not be used
  3596. * until rcu_assign_pointer(), so it's safe to increment
  3597. * it here.
  3598. */
  3599. ++new->current_threshold;
  3600. }
  3601. j++;
  3602. }
  3603. swap_buffers:
  3604. /* Swap primary and spare array */
  3605. thresholds->spare = thresholds->primary;
  3606. rcu_assign_pointer(thresholds->primary, new);
  3607. /* To be sure that nobody uses thresholds */
  3608. synchronize_rcu();
  3609. mutex_unlock(&memcg->thresholds_lock);
  3610. }
  3611. static int mem_cgroup_oom_register_event(struct cgroup *cgrp,
  3612. struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
  3613. {
  3614. struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
  3615. struct mem_cgroup_eventfd_list *event;
  3616. int type = MEMFILE_TYPE(cft->private);
  3617. BUG_ON(type != _OOM_TYPE);
  3618. event = kmalloc(sizeof(*event), GFP_KERNEL);
  3619. if (!event)
  3620. return -ENOMEM;
  3621. mutex_lock(&memcg_oom_mutex);
  3622. event->eventfd = eventfd;
  3623. list_add(&event->list, &memcg->oom_notify);
  3624. /* already in OOM ? */
  3625. if (atomic_read(&memcg->oom_lock))
  3626. eventfd_signal(eventfd, 1);
  3627. mutex_unlock(&memcg_oom_mutex);
  3628. return 0;
  3629. }
  3630. static void mem_cgroup_oom_unregister_event(struct cgroup *cgrp,
  3631. struct cftype *cft, struct eventfd_ctx *eventfd)
  3632. {
  3633. struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
  3634. struct mem_cgroup_eventfd_list *ev, *tmp;
  3635. int type = MEMFILE_TYPE(cft->private);
  3636. BUG_ON(type != _OOM_TYPE);
  3637. mutex_lock(&memcg_oom_mutex);
  3638. list_for_each_entry_safe(ev, tmp, &mem->oom_notify, list) {
  3639. if (ev->eventfd == eventfd) {
  3640. list_del(&ev->list);
  3641. kfree(ev);
  3642. }
  3643. }
  3644. mutex_unlock(&memcg_oom_mutex);
  3645. }
  3646. static int mem_cgroup_oom_control_read(struct cgroup *cgrp,
  3647. struct cftype *cft, struct cgroup_map_cb *cb)
  3648. {
  3649. struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
  3650. cb->fill(cb, "oom_kill_disable", mem->oom_kill_disable);
  3651. if (atomic_read(&mem->oom_lock))
  3652. cb->fill(cb, "under_oom", 1);
  3653. else
  3654. cb->fill(cb, "under_oom", 0);
  3655. return 0;
  3656. }
  3657. static int mem_cgroup_oom_control_write(struct cgroup *cgrp,
  3658. struct cftype *cft, u64 val)
  3659. {
  3660. struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
  3661. struct mem_cgroup *parent;
  3662. /* cannot set to root cgroup and only 0 and 1 are allowed */
  3663. if (!cgrp->parent || !((val == 0) || (val == 1)))
  3664. return -EINVAL;
  3665. parent = mem_cgroup_from_cont(cgrp->parent);
  3666. cgroup_lock();
  3667. /* oom-kill-disable is a flag for subhierarchy. */
  3668. if ((parent->use_hierarchy) ||
  3669. (mem->use_hierarchy && !list_empty(&cgrp->children))) {
  3670. cgroup_unlock();
  3671. return -EINVAL;
  3672. }
  3673. mem->oom_kill_disable = val;
  3674. if (!val)
  3675. memcg_oom_recover(mem);
  3676. cgroup_unlock();
  3677. return 0;
  3678. }
  3679. static struct cftype mem_cgroup_files[] = {
  3680. {
  3681. .name = "usage_in_bytes",
  3682. .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
  3683. .read_u64 = mem_cgroup_read,
  3684. .register_event = mem_cgroup_usage_register_event,
  3685. .unregister_event = mem_cgroup_usage_unregister_event,
  3686. },
  3687. {
  3688. .name = "max_usage_in_bytes",
  3689. .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
  3690. .trigger = mem_cgroup_reset,
  3691. .read_u64 = mem_cgroup_read,
  3692. },
  3693. {
  3694. .name = "limit_in_bytes",
  3695. .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
  3696. .write_string = mem_cgroup_write,
  3697. .read_u64 = mem_cgroup_read,
  3698. },
  3699. {
  3700. .name = "soft_limit_in_bytes",
  3701. .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
  3702. .write_string = mem_cgroup_write,
  3703. .read_u64 = mem_cgroup_read,
  3704. },
  3705. {
  3706. .name = "failcnt",
  3707. .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
  3708. .trigger = mem_cgroup_reset,
  3709. .read_u64 = mem_cgroup_read,
  3710. },
  3711. {
  3712. .name = "stat",
  3713. .read_map = mem_control_stat_show,
  3714. },
  3715. {
  3716. .name = "force_empty",
  3717. .trigger = mem_cgroup_force_empty_write,
  3718. },
  3719. {
  3720. .name = "use_hierarchy",
  3721. .write_u64 = mem_cgroup_hierarchy_write,
  3722. .read_u64 = mem_cgroup_hierarchy_read,
  3723. },
  3724. {
  3725. .name = "swappiness",
  3726. .read_u64 = mem_cgroup_swappiness_read,
  3727. .write_u64 = mem_cgroup_swappiness_write,
  3728. },
  3729. {
  3730. .name = "move_charge_at_immigrate",
  3731. .read_u64 = mem_cgroup_move_charge_read,
  3732. .write_u64 = mem_cgroup_move_charge_write,
  3733. },
  3734. {
  3735. .name = "oom_control",
  3736. .read_map = mem_cgroup_oom_control_read,
  3737. .write_u64 = mem_cgroup_oom_control_write,
  3738. .register_event = mem_cgroup_oom_register_event,
  3739. .unregister_event = mem_cgroup_oom_unregister_event,
  3740. .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
  3741. },
  3742. };
  3743. #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
  3744. static struct cftype memsw_cgroup_files[] = {
  3745. {
  3746. .name = "memsw.usage_in_bytes",
  3747. .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
  3748. .read_u64 = mem_cgroup_read,
  3749. .register_event = mem_cgroup_usage_register_event,
  3750. .unregister_event = mem_cgroup_usage_unregister_event,
  3751. },
  3752. {
  3753. .name = "memsw.max_usage_in_bytes",
  3754. .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
  3755. .trigger = mem_cgroup_reset,
  3756. .read_u64 = mem_cgroup_read,
  3757. },
  3758. {
  3759. .name = "memsw.limit_in_bytes",
  3760. .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
  3761. .write_string = mem_cgroup_write,
  3762. .read_u64 = mem_cgroup_read,
  3763. },
  3764. {
  3765. .name = "memsw.failcnt",
  3766. .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
  3767. .trigger = mem_cgroup_reset,
  3768. .read_u64 = mem_cgroup_read,
  3769. },
  3770. };
  3771. static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
  3772. {
  3773. if (!do_swap_account)
  3774. return 0;
  3775. return cgroup_add_files(cont, ss, memsw_cgroup_files,
  3776. ARRAY_SIZE(memsw_cgroup_files));
  3777. };
  3778. #else
  3779. static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
  3780. {
  3781. return 0;
  3782. }
  3783. #endif
  3784. static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
  3785. {
  3786. struct mem_cgroup_per_node *pn;
  3787. struct mem_cgroup_per_zone *mz;
  3788. enum lru_list l;
  3789. int zone, tmp = node;
  3790. /*
  3791. * This routine is called against possible nodes.
  3792. * But it's BUG to call kmalloc() against offline node.
  3793. *
  3794. * TODO: this routine can waste much memory for nodes which will
  3795. * never be onlined. It's better to use memory hotplug callback
  3796. * function.
  3797. */
  3798. if (!node_state(node, N_NORMAL_MEMORY))
  3799. tmp = -1;
  3800. pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
  3801. if (!pn)
  3802. return 1;
  3803. mem->info.nodeinfo[node] = pn;
  3804. for (zone = 0; zone < MAX_NR_ZONES; zone++) {
  3805. mz = &pn->zoneinfo[zone];
  3806. for_each_lru(l)
  3807. INIT_LIST_HEAD(&mz->lists[l]);
  3808. mz->usage_in_excess = 0;
  3809. mz->on_tree = false;
  3810. mz->mem = mem;
  3811. }
  3812. return 0;
  3813. }
  3814. static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
  3815. {
  3816. kfree(mem->info.nodeinfo[node]);
  3817. }
  3818. static struct mem_cgroup *mem_cgroup_alloc(void)
  3819. {
  3820. struct mem_cgroup *mem;
  3821. int size = sizeof(struct mem_cgroup);
  3822. /* Can be very big if MAX_NUMNODES is very big */
  3823. if (size < PAGE_SIZE)
  3824. mem = kzalloc(size, GFP_KERNEL);
  3825. else
  3826. mem = vzalloc(size);
  3827. if (!mem)
  3828. return NULL;
  3829. mem->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
  3830. if (!mem->stat)
  3831. goto out_free;
  3832. spin_lock_init(&mem->pcp_counter_lock);
  3833. return mem;
  3834. out_free:
  3835. if (size < PAGE_SIZE)
  3836. kfree(mem);
  3837. else
  3838. vfree(mem);
  3839. return NULL;
  3840. }
  3841. /*
  3842. * At destroying mem_cgroup, references from swap_cgroup can remain.
  3843. * (scanning all at force_empty is too costly...)
  3844. *
  3845. * Instead of clearing all references at force_empty, we remember
  3846. * the number of reference from swap_cgroup and free mem_cgroup when
  3847. * it goes down to 0.
  3848. *
  3849. * Removal of cgroup itself succeeds regardless of refs from swap.
  3850. */
  3851. static void __mem_cgroup_free(struct mem_cgroup *mem)
  3852. {
  3853. int node;
  3854. mem_cgroup_remove_from_trees(mem);
  3855. free_css_id(&mem_cgroup_subsys, &mem->css);
  3856. for_each_node_state(node, N_POSSIBLE)
  3857. free_mem_cgroup_per_zone_info(mem, node);
  3858. free_percpu(mem->stat);
  3859. if (sizeof(struct mem_cgroup) < PAGE_SIZE)
  3860. kfree(mem);
  3861. else
  3862. vfree(mem);
  3863. }
  3864. static void mem_cgroup_get(struct mem_cgroup *mem)
  3865. {
  3866. atomic_inc(&mem->refcnt);
  3867. }
  3868. static void __mem_cgroup_put(struct mem_cgroup *mem, int count)
  3869. {
  3870. if (atomic_sub_and_test(count, &mem->refcnt)) {
  3871. struct mem_cgroup *parent = parent_mem_cgroup(mem);
  3872. __mem_cgroup_free(mem);
  3873. if (parent)
  3874. mem_cgroup_put(parent);
  3875. }
  3876. }
  3877. static void mem_cgroup_put(struct mem_cgroup *mem)
  3878. {
  3879. __mem_cgroup_put(mem, 1);
  3880. }
  3881. /*
  3882. * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
  3883. */
  3884. static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem)
  3885. {
  3886. if (!mem->res.parent)
  3887. return NULL;
  3888. return mem_cgroup_from_res_counter(mem->res.parent, res);
  3889. }
  3890. #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
  3891. static void __init enable_swap_cgroup(void)
  3892. {
  3893. if (!mem_cgroup_disabled() && really_do_swap_account)
  3894. do_swap_account = 1;
  3895. }
  3896. #else
  3897. static void __init enable_swap_cgroup(void)
  3898. {
  3899. }
  3900. #endif
  3901. static int mem_cgroup_soft_limit_tree_init(void)
  3902. {
  3903. struct mem_cgroup_tree_per_node *rtpn;
  3904. struct mem_cgroup_tree_per_zone *rtpz;
  3905. int tmp, node, zone;
  3906. for_each_node_state(node, N_POSSIBLE) {
  3907. tmp = node;
  3908. if (!node_state(node, N_NORMAL_MEMORY))
  3909. tmp = -1;
  3910. rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp);
  3911. if (!rtpn)
  3912. return 1;
  3913. soft_limit_tree.rb_tree_per_node[node] = rtpn;
  3914. for (zone = 0; zone < MAX_NR_ZONES; zone++) {
  3915. rtpz = &rtpn->rb_tree_per_zone[zone];
  3916. rtpz->rb_root = RB_ROOT;
  3917. spin_lock_init(&rtpz->lock);
  3918. }
  3919. }
  3920. return 0;
  3921. }
  3922. static struct cgroup_subsys_state * __ref
  3923. mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
  3924. {
  3925. struct mem_cgroup *mem, *parent;
  3926. long error = -ENOMEM;
  3927. int node;
  3928. mem = mem_cgroup_alloc();
  3929. if (!mem)
  3930. return ERR_PTR(error);
  3931. for_each_node_state(node, N_POSSIBLE)
  3932. if (alloc_mem_cgroup_per_zone_info(mem, node))
  3933. goto free_out;
  3934. /* root ? */
  3935. if (cont->parent == NULL) {
  3936. int cpu;
  3937. enable_swap_cgroup();
  3938. parent = NULL;
  3939. root_mem_cgroup = mem;
  3940. if (mem_cgroup_soft_limit_tree_init())
  3941. goto free_out;
  3942. for_each_possible_cpu(cpu) {
  3943. struct memcg_stock_pcp *stock =
  3944. &per_cpu(memcg_stock, cpu);
  3945. INIT_WORK(&stock->work, drain_local_stock);
  3946. }
  3947. hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
  3948. } else {
  3949. parent = mem_cgroup_from_cont(cont->parent);
  3950. mem->use_hierarchy = parent->use_hierarchy;
  3951. mem->oom_kill_disable = parent->oom_kill_disable;
  3952. }
  3953. if (parent && parent->use_hierarchy) {
  3954. res_counter_init(&mem->res, &parent->res);
  3955. res_counter_init(&mem->memsw, &parent->memsw);
  3956. /*
  3957. * We increment refcnt of the parent to ensure that we can
  3958. * safely access it on res_counter_charge/uncharge.
  3959. * This refcnt will be decremented when freeing this
  3960. * mem_cgroup(see mem_cgroup_put).
  3961. */
  3962. mem_cgroup_get(parent);
  3963. } else {
  3964. res_counter_init(&mem->res, NULL);
  3965. res_counter_init(&mem->memsw, NULL);
  3966. }
  3967. mem->last_scanned_child = 0;
  3968. spin_lock_init(&mem->reclaim_param_lock);
  3969. INIT_LIST_HEAD(&mem->oom_notify);
  3970. if (parent)
  3971. mem->swappiness = get_swappiness(parent);
  3972. atomic_set(&mem->refcnt, 1);
  3973. mem->move_charge_at_immigrate = 0;
  3974. mutex_init(&mem->thresholds_lock);
  3975. return &mem->css;
  3976. free_out:
  3977. __mem_cgroup_free(mem);
  3978. root_mem_cgroup = NULL;
  3979. return ERR_PTR(error);
  3980. }
  3981. static int mem_cgroup_pre_destroy(struct cgroup_subsys *ss,
  3982. struct cgroup *cont)
  3983. {
  3984. struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
  3985. return mem_cgroup_force_empty(mem, false);
  3986. }
  3987. static void mem_cgroup_destroy(struct cgroup_subsys *ss,
  3988. struct cgroup *cont)
  3989. {
  3990. struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
  3991. mem_cgroup_put(mem);
  3992. }
  3993. static int mem_cgroup_populate(struct cgroup_subsys *ss,
  3994. struct cgroup *cont)
  3995. {
  3996. int ret;
  3997. ret = cgroup_add_files(cont, ss, mem_cgroup_files,
  3998. ARRAY_SIZE(mem_cgroup_files));
  3999. if (!ret)
  4000. ret = register_memsw_files(cont, ss);
  4001. return ret;
  4002. }
  4003. #ifdef CONFIG_MMU
  4004. /* Handlers for move charge at task migration. */
  4005. #define PRECHARGE_COUNT_AT_ONCE 256
  4006. static int mem_cgroup_do_precharge(unsigned long count)
  4007. {
  4008. int ret = 0;
  4009. int batch_count = PRECHARGE_COUNT_AT_ONCE;
  4010. struct mem_cgroup *mem = mc.to;
  4011. if (mem_cgroup_is_root(mem)) {
  4012. mc.precharge += count;
  4013. /* we don't need css_get for root */
  4014. return ret;
  4015. }
  4016. /* try to charge at once */
  4017. if (count > 1) {
  4018. struct res_counter *dummy;
  4019. /*
  4020. * "mem" cannot be under rmdir() because we've already checked
  4021. * by cgroup_lock_live_cgroup() that it is not removed and we
  4022. * are still under the same cgroup_mutex. So we can postpone
  4023. * css_get().
  4024. */
  4025. if (res_counter_charge(&mem->res, PAGE_SIZE * count, &dummy))
  4026. goto one_by_one;
  4027. if (do_swap_account && res_counter_charge(&mem->memsw,
  4028. PAGE_SIZE * count, &dummy)) {
  4029. res_counter_uncharge(&mem->res, PAGE_SIZE * count);
  4030. goto one_by_one;
  4031. }
  4032. mc.precharge += count;
  4033. return ret;
  4034. }
  4035. one_by_one:
  4036. /* fall back to one by one charge */
  4037. while (count--) {
  4038. if (signal_pending(current)) {
  4039. ret = -EINTR;
  4040. break;
  4041. }
  4042. if (!batch_count--) {
  4043. batch_count = PRECHARGE_COUNT_AT_ONCE;
  4044. cond_resched();
  4045. }
  4046. ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem, false,
  4047. PAGE_SIZE);
  4048. if (ret || !mem)
  4049. /* mem_cgroup_clear_mc() will do uncharge later */
  4050. return -ENOMEM;
  4051. mc.precharge++;
  4052. }
  4053. return ret;
  4054. }
  4055. /**
  4056. * is_target_pte_for_mc - check a pte whether it is valid for move charge
  4057. * @vma: the vma the pte to be checked belongs
  4058. * @addr: the address corresponding to the pte to be checked
  4059. * @ptent: the pte to be checked
  4060. * @target: the pointer the target page or swap ent will be stored(can be NULL)
  4061. *
  4062. * Returns
  4063. * 0(MC_TARGET_NONE): if the pte is not a target for move charge.
  4064. * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
  4065. * move charge. if @target is not NULL, the page is stored in target->page
  4066. * with extra refcnt got(Callers should handle it).
  4067. * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
  4068. * target for charge migration. if @target is not NULL, the entry is stored
  4069. * in target->ent.
  4070. *
  4071. * Called with pte lock held.
  4072. */
  4073. union mc_target {
  4074. struct page *page;
  4075. swp_entry_t ent;
  4076. };
  4077. enum mc_target_type {
  4078. MC_TARGET_NONE, /* not used */
  4079. MC_TARGET_PAGE,
  4080. MC_TARGET_SWAP,
  4081. };
  4082. static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
  4083. unsigned long addr, pte_t ptent)
  4084. {
  4085. struct page *page = vm_normal_page(vma, addr, ptent);
  4086. if (!page || !page_mapped(page))
  4087. return NULL;
  4088. if (PageAnon(page)) {
  4089. /* we don't move shared anon */
  4090. if (!move_anon() || page_mapcount(page) > 2)
  4091. return NULL;
  4092. } else if (!move_file())
  4093. /* we ignore mapcount for file pages */
  4094. return NULL;
  4095. if (!get_page_unless_zero(page))
  4096. return NULL;
  4097. return page;
  4098. }
  4099. static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
  4100. unsigned long addr, pte_t ptent, swp_entry_t *entry)
  4101. {
  4102. int usage_count;
  4103. struct page *page = NULL;
  4104. swp_entry_t ent = pte_to_swp_entry(ptent);
  4105. if (!move_anon() || non_swap_entry(ent))
  4106. return NULL;
  4107. usage_count = mem_cgroup_count_swap_user(ent, &page);
  4108. if (usage_count > 1) { /* we don't move shared anon */
  4109. if (page)
  4110. put_page(page);
  4111. return NULL;
  4112. }
  4113. if (do_swap_account)
  4114. entry->val = ent.val;
  4115. return page;
  4116. }
  4117. static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
  4118. unsigned long addr, pte_t ptent, swp_entry_t *entry)
  4119. {
  4120. struct page *page = NULL;
  4121. struct inode *inode;
  4122. struct address_space *mapping;
  4123. pgoff_t pgoff;
  4124. if (!vma->vm_file) /* anonymous vma */
  4125. return NULL;
  4126. if (!move_file())
  4127. return NULL;
  4128. inode = vma->vm_file->f_path.dentry->d_inode;
  4129. mapping = vma->vm_file->f_mapping;
  4130. if (pte_none(ptent))
  4131. pgoff = linear_page_index(vma, addr);
  4132. else /* pte_file(ptent) is true */
  4133. pgoff = pte_to_pgoff(ptent);
  4134. /* page is moved even if it's not RSS of this task(page-faulted). */
  4135. if (!mapping_cap_swap_backed(mapping)) { /* normal file */
  4136. page = find_get_page(mapping, pgoff);
  4137. } else { /* shmem/tmpfs file. we should take account of swap too. */
  4138. swp_entry_t ent;
  4139. mem_cgroup_get_shmem_target(inode, pgoff, &page, &ent);
  4140. if (do_swap_account)
  4141. entry->val = ent.val;
  4142. }
  4143. return page;
  4144. }
  4145. static int is_target_pte_for_mc(struct vm_area_struct *vma,
  4146. unsigned long addr, pte_t ptent, union mc_target *target)
  4147. {
  4148. struct page *page = NULL;
  4149. struct page_cgroup *pc;
  4150. int ret = 0;
  4151. swp_entry_t ent = { .val = 0 };
  4152. if (pte_present(ptent))
  4153. page = mc_handle_present_pte(vma, addr, ptent);
  4154. else if (is_swap_pte(ptent))
  4155. page = mc_handle_swap_pte(vma, addr, ptent, &ent);
  4156. else if (pte_none(ptent) || pte_file(ptent))
  4157. page = mc_handle_file_pte(vma, addr, ptent, &ent);
  4158. if (!page && !ent.val)
  4159. return 0;
  4160. if (page) {
  4161. pc = lookup_page_cgroup(page);
  4162. /*
  4163. * Do only loose check w/o page_cgroup lock.
  4164. * mem_cgroup_move_account() checks the pc is valid or not under
  4165. * the lock.
  4166. */
  4167. if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
  4168. ret = MC_TARGET_PAGE;
  4169. if (target)
  4170. target->page = page;
  4171. }
  4172. if (!ret || !target)
  4173. put_page(page);
  4174. }
  4175. /* There is a swap entry and a page doesn't exist or isn't charged */
  4176. if (ent.val && !ret &&
  4177. css_id(&mc.from->css) == lookup_swap_cgroup(ent)) {
  4178. ret = MC_TARGET_SWAP;
  4179. if (target)
  4180. target->ent = ent;
  4181. }
  4182. return ret;
  4183. }
  4184. static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
  4185. unsigned long addr, unsigned long end,
  4186. struct mm_walk *walk)
  4187. {
  4188. struct vm_area_struct *vma = walk->private;
  4189. pte_t *pte;
  4190. spinlock_t *ptl;
  4191. split_huge_page_pmd(walk->mm, pmd);
  4192. pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
  4193. for (; addr != end; pte++, addr += PAGE_SIZE)
  4194. if (is_target_pte_for_mc(vma, addr, *pte, NULL))
  4195. mc.precharge++; /* increment precharge temporarily */
  4196. pte_unmap_unlock(pte - 1, ptl);
  4197. cond_resched();
  4198. return 0;
  4199. }
  4200. static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
  4201. {
  4202. unsigned long precharge;
  4203. struct vm_area_struct *vma;
  4204. down_read(&mm->mmap_sem);
  4205. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  4206. struct mm_walk mem_cgroup_count_precharge_walk = {
  4207. .pmd_entry = mem_cgroup_count_precharge_pte_range,
  4208. .mm = mm,
  4209. .private = vma,
  4210. };
  4211. if (is_vm_hugetlb_page(vma))
  4212. continue;
  4213. walk_page_range(vma->vm_start, vma->vm_end,
  4214. &mem_cgroup_count_precharge_walk);
  4215. }
  4216. up_read(&mm->mmap_sem);
  4217. precharge = mc.precharge;
  4218. mc.precharge = 0;
  4219. return precharge;
  4220. }
  4221. static int mem_cgroup_precharge_mc(struct mm_struct *mm)
  4222. {
  4223. unsigned long precharge = mem_cgroup_count_precharge(mm);
  4224. VM_BUG_ON(mc.moving_task);
  4225. mc.moving_task = current;
  4226. return mem_cgroup_do_precharge(precharge);
  4227. }
  4228. /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
  4229. static void __mem_cgroup_clear_mc(void)
  4230. {
  4231. struct mem_cgroup *from = mc.from;
  4232. struct mem_cgroup *to = mc.to;
  4233. /* we must uncharge all the leftover precharges from mc.to */
  4234. if (mc.precharge) {
  4235. __mem_cgroup_cancel_charge(mc.to, mc.precharge);
  4236. mc.precharge = 0;
  4237. }
  4238. /*
  4239. * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
  4240. * we must uncharge here.
  4241. */
  4242. if (mc.moved_charge) {
  4243. __mem_cgroup_cancel_charge(mc.from, mc.moved_charge);
  4244. mc.moved_charge = 0;
  4245. }
  4246. /* we must fixup refcnts and charges */
  4247. if (mc.moved_swap) {
  4248. /* uncharge swap account from the old cgroup */
  4249. if (!mem_cgroup_is_root(mc.from))
  4250. res_counter_uncharge(&mc.from->memsw,
  4251. PAGE_SIZE * mc.moved_swap);
  4252. __mem_cgroup_put(mc.from, mc.moved_swap);
  4253. if (!mem_cgroup_is_root(mc.to)) {
  4254. /*
  4255. * we charged both to->res and to->memsw, so we should
  4256. * uncharge to->res.
  4257. */
  4258. res_counter_uncharge(&mc.to->res,
  4259. PAGE_SIZE * mc.moved_swap);
  4260. }
  4261. /* we've already done mem_cgroup_get(mc.to) */
  4262. mc.moved_swap = 0;
  4263. }
  4264. memcg_oom_recover(from);
  4265. memcg_oom_recover(to);
  4266. wake_up_all(&mc.waitq);
  4267. }
  4268. static void mem_cgroup_clear_mc(void)
  4269. {
  4270. struct mem_cgroup *from = mc.from;
  4271. /*
  4272. * we must clear moving_task before waking up waiters at the end of
  4273. * task migration.
  4274. */
  4275. mc.moving_task = NULL;
  4276. __mem_cgroup_clear_mc();
  4277. spin_lock(&mc.lock);
  4278. mc.from = NULL;
  4279. mc.to = NULL;
  4280. spin_unlock(&mc.lock);
  4281. mem_cgroup_end_move(from);
  4282. }
  4283. static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
  4284. struct cgroup *cgroup,
  4285. struct task_struct *p,
  4286. bool threadgroup)
  4287. {
  4288. int ret = 0;
  4289. struct mem_cgroup *mem = mem_cgroup_from_cont(cgroup);
  4290. if (mem->move_charge_at_immigrate) {
  4291. struct mm_struct *mm;
  4292. struct mem_cgroup *from = mem_cgroup_from_task(p);
  4293. VM_BUG_ON(from == mem);
  4294. mm = get_task_mm(p);
  4295. if (!mm)
  4296. return 0;
  4297. /* We move charges only when we move a owner of the mm */
  4298. if (mm->owner == p) {
  4299. VM_BUG_ON(mc.from);
  4300. VM_BUG_ON(mc.to);
  4301. VM_BUG_ON(mc.precharge);
  4302. VM_BUG_ON(mc.moved_charge);
  4303. VM_BUG_ON(mc.moved_swap);
  4304. mem_cgroup_start_move(from);
  4305. spin_lock(&mc.lock);
  4306. mc.from = from;
  4307. mc.to = mem;
  4308. spin_unlock(&mc.lock);
  4309. /* We set mc.moving_task later */
  4310. ret = mem_cgroup_precharge_mc(mm);
  4311. if (ret)
  4312. mem_cgroup_clear_mc();
  4313. }
  4314. mmput(mm);
  4315. }
  4316. return ret;
  4317. }
  4318. static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
  4319. struct cgroup *cgroup,
  4320. struct task_struct *p,
  4321. bool threadgroup)
  4322. {
  4323. mem_cgroup_clear_mc();
  4324. }
  4325. static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
  4326. unsigned long addr, unsigned long end,
  4327. struct mm_walk *walk)
  4328. {
  4329. int ret = 0;
  4330. struct vm_area_struct *vma = walk->private;
  4331. pte_t *pte;
  4332. spinlock_t *ptl;
  4333. split_huge_page_pmd(walk->mm, pmd);
  4334. retry:
  4335. pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
  4336. for (; addr != end; addr += PAGE_SIZE) {
  4337. pte_t ptent = *(pte++);
  4338. union mc_target target;
  4339. int type;
  4340. struct page *page;
  4341. struct page_cgroup *pc;
  4342. swp_entry_t ent;
  4343. if (!mc.precharge)
  4344. break;
  4345. type = is_target_pte_for_mc(vma, addr, ptent, &target);
  4346. switch (type) {
  4347. case MC_TARGET_PAGE:
  4348. page = target.page;
  4349. if (isolate_lru_page(page))
  4350. goto put;
  4351. pc = lookup_page_cgroup(page);
  4352. if (!mem_cgroup_move_account(pc,
  4353. mc.from, mc.to, false, PAGE_SIZE)) {
  4354. mc.precharge--;
  4355. /* we uncharge from mc.from later. */
  4356. mc.moved_charge++;
  4357. }
  4358. putback_lru_page(page);
  4359. put: /* is_target_pte_for_mc() gets the page */
  4360. put_page(page);
  4361. break;
  4362. case MC_TARGET_SWAP:
  4363. ent = target.ent;
  4364. if (!mem_cgroup_move_swap_account(ent,
  4365. mc.from, mc.to, false)) {
  4366. mc.precharge--;
  4367. /* we fixup refcnts and charges later. */
  4368. mc.moved_swap++;
  4369. }
  4370. break;
  4371. default:
  4372. break;
  4373. }
  4374. }
  4375. pte_unmap_unlock(pte - 1, ptl);
  4376. cond_resched();
  4377. if (addr != end) {
  4378. /*
  4379. * We have consumed all precharges we got in can_attach().
  4380. * We try charge one by one, but don't do any additional
  4381. * charges to mc.to if we have failed in charge once in attach()
  4382. * phase.
  4383. */
  4384. ret = mem_cgroup_do_precharge(1);
  4385. if (!ret)
  4386. goto retry;
  4387. }
  4388. return ret;
  4389. }
  4390. static void mem_cgroup_move_charge(struct mm_struct *mm)
  4391. {
  4392. struct vm_area_struct *vma;
  4393. lru_add_drain_all();
  4394. retry:
  4395. if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
  4396. /*
  4397. * Someone who are holding the mmap_sem might be waiting in
  4398. * waitq. So we cancel all extra charges, wake up all waiters,
  4399. * and retry. Because we cancel precharges, we might not be able
  4400. * to move enough charges, but moving charge is a best-effort
  4401. * feature anyway, so it wouldn't be a big problem.
  4402. */
  4403. __mem_cgroup_clear_mc();
  4404. cond_resched();
  4405. goto retry;
  4406. }
  4407. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  4408. int ret;
  4409. struct mm_walk mem_cgroup_move_charge_walk = {
  4410. .pmd_entry = mem_cgroup_move_charge_pte_range,
  4411. .mm = mm,
  4412. .private = vma,
  4413. };
  4414. if (is_vm_hugetlb_page(vma))
  4415. continue;
  4416. ret = walk_page_range(vma->vm_start, vma->vm_end,
  4417. &mem_cgroup_move_charge_walk);
  4418. if (ret)
  4419. /*
  4420. * means we have consumed all precharges and failed in
  4421. * doing additional charge. Just abandon here.
  4422. */
  4423. break;
  4424. }
  4425. up_read(&mm->mmap_sem);
  4426. }
  4427. static void mem_cgroup_move_task(struct cgroup_subsys *ss,
  4428. struct cgroup *cont,
  4429. struct cgroup *old_cont,
  4430. struct task_struct *p,
  4431. bool threadgroup)
  4432. {
  4433. struct mm_struct *mm;
  4434. if (!mc.to)
  4435. /* no need to move charge */
  4436. return;
  4437. mm = get_task_mm(p);
  4438. if (mm) {
  4439. mem_cgroup_move_charge(mm);
  4440. mmput(mm);
  4441. }
  4442. mem_cgroup_clear_mc();
  4443. }
  4444. #else /* !CONFIG_MMU */
  4445. static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
  4446. struct cgroup *cgroup,
  4447. struct task_struct *p,
  4448. bool threadgroup)
  4449. {
  4450. return 0;
  4451. }
  4452. static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
  4453. struct cgroup *cgroup,
  4454. struct task_struct *p,
  4455. bool threadgroup)
  4456. {
  4457. }
  4458. static void mem_cgroup_move_task(struct cgroup_subsys *ss,
  4459. struct cgroup *cont,
  4460. struct cgroup *old_cont,
  4461. struct task_struct *p,
  4462. bool threadgroup)
  4463. {
  4464. }
  4465. #endif
  4466. struct cgroup_subsys mem_cgroup_subsys = {
  4467. .name = "memory",
  4468. .subsys_id = mem_cgroup_subsys_id,
  4469. .create = mem_cgroup_create,
  4470. .pre_destroy = mem_cgroup_pre_destroy,
  4471. .destroy = mem_cgroup_destroy,
  4472. .populate = mem_cgroup_populate,
  4473. .can_attach = mem_cgroup_can_attach,
  4474. .cancel_attach = mem_cgroup_cancel_attach,
  4475. .attach = mem_cgroup_move_task,
  4476. .early_init = 0,
  4477. .use_id = 1,
  4478. };
  4479. #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
  4480. static int __init enable_swap_account(char *s)
  4481. {
  4482. /* consider enabled if no parameter or 1 is given */
  4483. if (!(*s) || !strcmp(s, "=1"))
  4484. really_do_swap_account = 1;
  4485. else if (!strcmp(s, "=0"))
  4486. really_do_swap_account = 0;
  4487. return 1;
  4488. }
  4489. __setup("swapaccount", enable_swap_account);
  4490. static int __init disable_swap_account(char *s)
  4491. {
  4492. printk_once("noswapaccount is deprecated and will be removed in 2.6.40. Use swapaccount=0 instead\n");
  4493. enable_swap_account("=0");
  4494. return 1;
  4495. }
  4496. __setup("noswapaccount", disable_swap_account);
  4497. #endif