memcontrol.c 180 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853
  1. /* memcontrol.c - Memory Controller
  2. *
  3. * Copyright IBM Corporation, 2007
  4. * Author Balbir Singh <balbir@linux.vnet.ibm.com>
  5. *
  6. * Copyright 2007 OpenVZ SWsoft Inc
  7. * Author: Pavel Emelianov <xemul@openvz.org>
  8. *
  9. * Memory thresholds
  10. * Copyright (C) 2009 Nokia Corporation
  11. * Author: Kirill A. Shutemov
  12. *
  13. * Kernel Memory Controller
  14. * Copyright (C) 2012 Parallels Inc. and Google Inc.
  15. * Authors: Glauber Costa and Suleiman Souhlal
  16. *
  17. * This program is free software; you can redistribute it and/or modify
  18. * it under the terms of the GNU General Public License as published by
  19. * the Free Software Foundation; either version 2 of the License, or
  20. * (at your option) any later version.
  21. *
  22. * This program is distributed in the hope that it will be useful,
  23. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  24. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  25. * GNU General Public License for more details.
  26. */
  27. #include <linux/res_counter.h>
  28. #include <linux/memcontrol.h>
  29. #include <linux/cgroup.h>
  30. #include <linux/mm.h>
  31. #include <linux/hugetlb.h>
  32. #include <linux/pagemap.h>
  33. #include <linux/smp.h>
  34. #include <linux/page-flags.h>
  35. #include <linux/backing-dev.h>
  36. #include <linux/bit_spinlock.h>
  37. #include <linux/rcupdate.h>
  38. #include <linux/limits.h>
  39. #include <linux/export.h>
  40. #include <linux/mutex.h>
  41. #include <linux/rbtree.h>
  42. #include <linux/slab.h>
  43. #include <linux/swap.h>
  44. #include <linux/swapops.h>
  45. #include <linux/spinlock.h>
  46. #include <linux/eventfd.h>
  47. #include <linux/sort.h>
  48. #include <linux/fs.h>
  49. #include <linux/seq_file.h>
  50. #include <linux/vmalloc.h>
  51. #include <linux/mm_inline.h>
  52. #include <linux/page_cgroup.h>
  53. #include <linux/cpu.h>
  54. #include <linux/oom.h>
  55. #include "internal.h"
  56. #include <net/sock.h>
  57. #include <net/ip.h>
  58. #include <net/tcp_memcontrol.h>
  59. #include <asm/uaccess.h>
  60. #include <trace/events/vmscan.h>
  61. struct cgroup_subsys mem_cgroup_subsys __read_mostly;
  62. EXPORT_SYMBOL(mem_cgroup_subsys);
  63. #define MEM_CGROUP_RECLAIM_RETRIES 5
  64. static struct mem_cgroup *root_mem_cgroup __read_mostly;
  65. #ifdef CONFIG_MEMCG_SWAP
  66. /* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */
  67. int do_swap_account __read_mostly;
  68. /* for remember boot option*/
  69. #ifdef CONFIG_MEMCG_SWAP_ENABLED
  70. static int really_do_swap_account __initdata = 1;
  71. #else
  72. static int really_do_swap_account __initdata = 0;
  73. #endif
  74. #else
  75. #define do_swap_account 0
  76. #endif
  77. /*
  78. * Statistics for memory cgroup.
  79. */
  80. enum mem_cgroup_stat_index {
  81. /*
  82. * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
  83. */
  84. MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */
  85. MEM_CGROUP_STAT_RSS, /* # of pages charged as anon rss */
  86. MEM_CGROUP_STAT_FILE_MAPPED, /* # of pages charged as file rss */
  87. MEM_CGROUP_STAT_SWAP, /* # of pages, swapped out */
  88. MEM_CGROUP_STAT_NSTATS,
  89. };
  90. static const char * const mem_cgroup_stat_names[] = {
  91. "cache",
  92. "rss",
  93. "mapped_file",
  94. "swap",
  95. };
  96. enum mem_cgroup_events_index {
  97. MEM_CGROUP_EVENTS_PGPGIN, /* # of pages paged in */
  98. MEM_CGROUP_EVENTS_PGPGOUT, /* # of pages paged out */
  99. MEM_CGROUP_EVENTS_PGFAULT, /* # of page-faults */
  100. MEM_CGROUP_EVENTS_PGMAJFAULT, /* # of major page-faults */
  101. MEM_CGROUP_EVENTS_NSTATS,
  102. };
  103. static const char * const mem_cgroup_events_names[] = {
  104. "pgpgin",
  105. "pgpgout",
  106. "pgfault",
  107. "pgmajfault",
  108. };
  109. static const char * const mem_cgroup_lru_names[] = {
  110. "inactive_anon",
  111. "active_anon",
  112. "inactive_file",
  113. "active_file",
  114. "unevictable",
  115. };
  116. /*
  117. * Per memcg event counter is incremented at every pagein/pageout. With THP,
  118. * it will be incremated by the number of pages. This counter is used for
  119. * for trigger some periodic events. This is straightforward and better
  120. * than using jiffies etc. to handle periodic memcg event.
  121. */
  122. enum mem_cgroup_events_target {
  123. MEM_CGROUP_TARGET_THRESH,
  124. MEM_CGROUP_TARGET_SOFTLIMIT,
  125. MEM_CGROUP_TARGET_NUMAINFO,
  126. MEM_CGROUP_NTARGETS,
  127. };
  128. #define THRESHOLDS_EVENTS_TARGET 128
  129. #define SOFTLIMIT_EVENTS_TARGET 1024
  130. #define NUMAINFO_EVENTS_TARGET 1024
  131. struct mem_cgroup_stat_cpu {
  132. long count[MEM_CGROUP_STAT_NSTATS];
  133. unsigned long events[MEM_CGROUP_EVENTS_NSTATS];
  134. unsigned long nr_page_events;
  135. unsigned long targets[MEM_CGROUP_NTARGETS];
  136. };
  137. struct mem_cgroup_reclaim_iter {
  138. /* css_id of the last scanned hierarchy member */
  139. int position;
  140. /* scan generation, increased every round-trip */
  141. unsigned int generation;
  142. };
  143. /*
  144. * per-zone information in memory controller.
  145. */
  146. struct mem_cgroup_per_zone {
  147. struct lruvec lruvec;
  148. unsigned long lru_size[NR_LRU_LISTS];
  149. struct mem_cgroup_reclaim_iter reclaim_iter[DEF_PRIORITY + 1];
  150. struct rb_node tree_node; /* RB tree node */
  151. unsigned long long usage_in_excess;/* Set to the value by which */
  152. /* the soft limit is exceeded*/
  153. bool on_tree;
  154. struct mem_cgroup *memcg; /* Back pointer, we cannot */
  155. /* use container_of */
  156. };
  157. struct mem_cgroup_per_node {
  158. struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
  159. };
  160. struct mem_cgroup_lru_info {
  161. struct mem_cgroup_per_node *nodeinfo[0];
  162. };
  163. /*
  164. * Cgroups above their limits are maintained in a RB-Tree, independent of
  165. * their hierarchy representation
  166. */
  167. struct mem_cgroup_tree_per_zone {
  168. struct rb_root rb_root;
  169. spinlock_t lock;
  170. };
  171. struct mem_cgroup_tree_per_node {
  172. struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
  173. };
  174. struct mem_cgroup_tree {
  175. struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
  176. };
  177. static struct mem_cgroup_tree soft_limit_tree __read_mostly;
  178. struct mem_cgroup_threshold {
  179. struct eventfd_ctx *eventfd;
  180. u64 threshold;
  181. };
  182. /* For threshold */
  183. struct mem_cgroup_threshold_ary {
  184. /* An array index points to threshold just below or equal to usage. */
  185. int current_threshold;
  186. /* Size of entries[] */
  187. unsigned int size;
  188. /* Array of thresholds */
  189. struct mem_cgroup_threshold entries[0];
  190. };
  191. struct mem_cgroup_thresholds {
  192. /* Primary thresholds array */
  193. struct mem_cgroup_threshold_ary *primary;
  194. /*
  195. * Spare threshold array.
  196. * This is needed to make mem_cgroup_unregister_event() "never fail".
  197. * It must be able to store at least primary->size - 1 entries.
  198. */
  199. struct mem_cgroup_threshold_ary *spare;
  200. };
  201. /* for OOM */
  202. struct mem_cgroup_eventfd_list {
  203. struct list_head list;
  204. struct eventfd_ctx *eventfd;
  205. };
  206. static void mem_cgroup_threshold(struct mem_cgroup *memcg);
  207. static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
  208. /*
  209. * The memory controller data structure. The memory controller controls both
  210. * page cache and RSS per cgroup. We would eventually like to provide
  211. * statistics based on the statistics developed by Rik Van Riel for clock-pro,
  212. * to help the administrator determine what knobs to tune.
  213. *
  214. * TODO: Add a water mark for the memory controller. Reclaim will begin when
  215. * we hit the water mark. May be even add a low water mark, such that
  216. * no reclaim occurs from a cgroup at it's low water mark, this is
  217. * a feature that will be implemented much later in the future.
  218. */
  219. struct mem_cgroup {
  220. struct cgroup_subsys_state css;
  221. /*
  222. * the counter to account for memory usage
  223. */
  224. struct res_counter res;
  225. union {
  226. /*
  227. * the counter to account for mem+swap usage.
  228. */
  229. struct res_counter memsw;
  230. /*
  231. * rcu_freeing is used only when freeing struct mem_cgroup,
  232. * so put it into a union to avoid wasting more memory.
  233. * It must be disjoint from the css field. It could be
  234. * in a union with the res field, but res plays a much
  235. * larger part in mem_cgroup life than memsw, and might
  236. * be of interest, even at time of free, when debugging.
  237. * So share rcu_head with the less interesting memsw.
  238. */
  239. struct rcu_head rcu_freeing;
  240. /*
  241. * We also need some space for a worker in deferred freeing.
  242. * By the time we call it, rcu_freeing is no longer in use.
  243. */
  244. struct work_struct work_freeing;
  245. };
  246. /*
  247. * the counter to account for kernel memory usage.
  248. */
  249. struct res_counter kmem;
  250. /*
  251. * Should the accounting and control be hierarchical, per subtree?
  252. */
  253. bool use_hierarchy;
  254. unsigned long kmem_account_flags; /* See KMEM_ACCOUNTED_*, below */
  255. bool oom_lock;
  256. atomic_t under_oom;
  257. atomic_t refcnt;
  258. int swappiness;
  259. /* OOM-Killer disable */
  260. int oom_kill_disable;
  261. /* set when res.limit == memsw.limit */
  262. bool memsw_is_minimum;
  263. /* protect arrays of thresholds */
  264. struct mutex thresholds_lock;
  265. /* thresholds for memory usage. RCU-protected */
  266. struct mem_cgroup_thresholds thresholds;
  267. /* thresholds for mem+swap usage. RCU-protected */
  268. struct mem_cgroup_thresholds memsw_thresholds;
  269. /* For oom notifier event fd */
  270. struct list_head oom_notify;
  271. /*
  272. * Should we move charges of a task when a task is moved into this
  273. * mem_cgroup ? And what type of charges should we move ?
  274. */
  275. unsigned long move_charge_at_immigrate;
  276. /*
  277. * set > 0 if pages under this cgroup are moving to other cgroup.
  278. */
  279. atomic_t moving_account;
  280. /* taken only while moving_account > 0 */
  281. spinlock_t move_lock;
  282. /*
  283. * percpu counter.
  284. */
  285. struct mem_cgroup_stat_cpu __percpu *stat;
  286. /*
  287. * used when a cpu is offlined or other synchronizations
  288. * See mem_cgroup_read_stat().
  289. */
  290. struct mem_cgroup_stat_cpu nocpu_base;
  291. spinlock_t pcp_counter_lock;
  292. #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_INET)
  293. struct tcp_memcontrol tcp_mem;
  294. #endif
  295. #if defined(CONFIG_MEMCG_KMEM)
  296. /* analogous to slab_common's slab_caches list. per-memcg */
  297. struct list_head memcg_slab_caches;
  298. /* Not a spinlock, we can take a lot of time walking the list */
  299. struct mutex slab_caches_mutex;
  300. /* Index in the kmem_cache->memcg_params->memcg_caches array */
  301. int kmemcg_id;
  302. #endif
  303. int last_scanned_node;
  304. #if MAX_NUMNODES > 1
  305. nodemask_t scan_nodes;
  306. atomic_t numainfo_events;
  307. atomic_t numainfo_updating;
  308. #endif
  309. /*
  310. * Per cgroup active and inactive list, similar to the
  311. * per zone LRU lists.
  312. *
  313. * WARNING: This has to be the last element of the struct. Don't
  314. * add new fields after this point.
  315. */
  316. struct mem_cgroup_lru_info info;
  317. };
  318. static size_t memcg_size(void)
  319. {
  320. return sizeof(struct mem_cgroup) +
  321. nr_node_ids * sizeof(struct mem_cgroup_per_node);
  322. }
  323. /* internal only representation about the status of kmem accounting. */
  324. enum {
  325. KMEM_ACCOUNTED_ACTIVE = 0, /* accounted by this cgroup itself */
  326. KMEM_ACCOUNTED_ACTIVATED, /* static key enabled. */
  327. KMEM_ACCOUNTED_DEAD, /* dead memcg with pending kmem charges */
  328. };
  329. /* We account when limit is on, but only after call sites are patched */
  330. #define KMEM_ACCOUNTED_MASK \
  331. ((1 << KMEM_ACCOUNTED_ACTIVE) | (1 << KMEM_ACCOUNTED_ACTIVATED))
  332. #ifdef CONFIG_MEMCG_KMEM
  333. static inline void memcg_kmem_set_active(struct mem_cgroup *memcg)
  334. {
  335. set_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags);
  336. }
  337. static bool memcg_kmem_is_active(struct mem_cgroup *memcg)
  338. {
  339. return test_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags);
  340. }
  341. static void memcg_kmem_set_activated(struct mem_cgroup *memcg)
  342. {
  343. set_bit(KMEM_ACCOUNTED_ACTIVATED, &memcg->kmem_account_flags);
  344. }
  345. static void memcg_kmem_clear_activated(struct mem_cgroup *memcg)
  346. {
  347. clear_bit(KMEM_ACCOUNTED_ACTIVATED, &memcg->kmem_account_flags);
  348. }
  349. static void memcg_kmem_mark_dead(struct mem_cgroup *memcg)
  350. {
  351. if (test_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags))
  352. set_bit(KMEM_ACCOUNTED_DEAD, &memcg->kmem_account_flags);
  353. }
  354. static bool memcg_kmem_test_and_clear_dead(struct mem_cgroup *memcg)
  355. {
  356. return test_and_clear_bit(KMEM_ACCOUNTED_DEAD,
  357. &memcg->kmem_account_flags);
  358. }
  359. #endif
  360. /* Stuffs for move charges at task migration. */
  361. /*
  362. * Types of charges to be moved. "move_charge_at_immitgrate" and
  363. * "immigrate_flags" are treated as a left-shifted bitmap of these types.
  364. */
  365. enum move_type {
  366. MOVE_CHARGE_TYPE_ANON, /* private anonymous page and swap of it */
  367. MOVE_CHARGE_TYPE_FILE, /* file page(including tmpfs) and swap of it */
  368. NR_MOVE_TYPE,
  369. };
  370. /* "mc" and its members are protected by cgroup_mutex */
  371. static struct move_charge_struct {
  372. spinlock_t lock; /* for from, to */
  373. struct mem_cgroup *from;
  374. struct mem_cgroup *to;
  375. unsigned long immigrate_flags;
  376. unsigned long precharge;
  377. unsigned long moved_charge;
  378. unsigned long moved_swap;
  379. struct task_struct *moving_task; /* a task moving charges */
  380. wait_queue_head_t waitq; /* a waitq for other context */
  381. } mc = {
  382. .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
  383. .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
  384. };
  385. static bool move_anon(void)
  386. {
  387. return test_bit(MOVE_CHARGE_TYPE_ANON, &mc.immigrate_flags);
  388. }
  389. static bool move_file(void)
  390. {
  391. return test_bit(MOVE_CHARGE_TYPE_FILE, &mc.immigrate_flags);
  392. }
  393. /*
  394. * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
  395. * limit reclaim to prevent infinite loops, if they ever occur.
  396. */
  397. #define MEM_CGROUP_MAX_RECLAIM_LOOPS 100
  398. #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
  399. enum charge_type {
  400. MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
  401. MEM_CGROUP_CHARGE_TYPE_ANON,
  402. MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
  403. MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */
  404. NR_CHARGE_TYPE,
  405. };
  406. /* for encoding cft->private value on file */
  407. enum res_type {
  408. _MEM,
  409. _MEMSWAP,
  410. _OOM_TYPE,
  411. _KMEM,
  412. };
  413. #define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))
  414. #define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff)
  415. #define MEMFILE_ATTR(val) ((val) & 0xffff)
  416. /* Used for OOM nofiier */
  417. #define OOM_CONTROL (0)
  418. /*
  419. * Reclaim flags for mem_cgroup_hierarchical_reclaim
  420. */
  421. #define MEM_CGROUP_RECLAIM_NOSWAP_BIT 0x0
  422. #define MEM_CGROUP_RECLAIM_NOSWAP (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
  423. #define MEM_CGROUP_RECLAIM_SHRINK_BIT 0x1
  424. #define MEM_CGROUP_RECLAIM_SHRINK (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
  425. /*
  426. * The memcg_create_mutex will be held whenever a new cgroup is created.
  427. * As a consequence, any change that needs to protect against new child cgroups
  428. * appearing has to hold it as well.
  429. */
  430. static DEFINE_MUTEX(memcg_create_mutex);
  431. static void mem_cgroup_get(struct mem_cgroup *memcg);
  432. static void mem_cgroup_put(struct mem_cgroup *memcg);
  433. static inline
  434. struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *s)
  435. {
  436. return container_of(s, struct mem_cgroup, css);
  437. }
  438. static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
  439. {
  440. return (memcg == root_mem_cgroup);
  441. }
  442. /* Writing them here to avoid exposing memcg's inner layout */
  443. #if defined(CONFIG_INET) && defined(CONFIG_MEMCG_KMEM)
  444. void sock_update_memcg(struct sock *sk)
  445. {
  446. if (mem_cgroup_sockets_enabled) {
  447. struct mem_cgroup *memcg;
  448. struct cg_proto *cg_proto;
  449. BUG_ON(!sk->sk_prot->proto_cgroup);
  450. /* Socket cloning can throw us here with sk_cgrp already
  451. * filled. It won't however, necessarily happen from
  452. * process context. So the test for root memcg given
  453. * the current task's memcg won't help us in this case.
  454. *
  455. * Respecting the original socket's memcg is a better
  456. * decision in this case.
  457. */
  458. if (sk->sk_cgrp) {
  459. BUG_ON(mem_cgroup_is_root(sk->sk_cgrp->memcg));
  460. mem_cgroup_get(sk->sk_cgrp->memcg);
  461. return;
  462. }
  463. rcu_read_lock();
  464. memcg = mem_cgroup_from_task(current);
  465. cg_proto = sk->sk_prot->proto_cgroup(memcg);
  466. if (!mem_cgroup_is_root(memcg) && memcg_proto_active(cg_proto)) {
  467. mem_cgroup_get(memcg);
  468. sk->sk_cgrp = cg_proto;
  469. }
  470. rcu_read_unlock();
  471. }
  472. }
  473. EXPORT_SYMBOL(sock_update_memcg);
  474. void sock_release_memcg(struct sock *sk)
  475. {
  476. if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
  477. struct mem_cgroup *memcg;
  478. WARN_ON(!sk->sk_cgrp->memcg);
  479. memcg = sk->sk_cgrp->memcg;
  480. mem_cgroup_put(memcg);
  481. }
  482. }
  483. struct cg_proto *tcp_proto_cgroup(struct mem_cgroup *memcg)
  484. {
  485. if (!memcg || mem_cgroup_is_root(memcg))
  486. return NULL;
  487. return &memcg->tcp_mem.cg_proto;
  488. }
  489. EXPORT_SYMBOL(tcp_proto_cgroup);
  490. static void disarm_sock_keys(struct mem_cgroup *memcg)
  491. {
  492. if (!memcg_proto_activated(&memcg->tcp_mem.cg_proto))
  493. return;
  494. static_key_slow_dec(&memcg_socket_limit_enabled);
  495. }
  496. #else
  497. static void disarm_sock_keys(struct mem_cgroup *memcg)
  498. {
  499. }
  500. #endif
  501. #ifdef CONFIG_MEMCG_KMEM
  502. /*
  503. * This will be the memcg's index in each cache's ->memcg_params->memcg_caches.
  504. * There are two main reasons for not using the css_id for this:
  505. * 1) this works better in sparse environments, where we have a lot of memcgs,
  506. * but only a few kmem-limited. Or also, if we have, for instance, 200
  507. * memcgs, and none but the 200th is kmem-limited, we'd have to have a
  508. * 200 entry array for that.
  509. *
  510. * 2) In order not to violate the cgroup API, we would like to do all memory
  511. * allocation in ->create(). At that point, we haven't yet allocated the
  512. * css_id. Having a separate index prevents us from messing with the cgroup
  513. * core for this
  514. *
  515. * The current size of the caches array is stored in
  516. * memcg_limited_groups_array_size. It will double each time we have to
  517. * increase it.
  518. */
  519. static DEFINE_IDA(kmem_limited_groups);
  520. int memcg_limited_groups_array_size;
  521. /*
  522. * MIN_SIZE is different than 1, because we would like to avoid going through
  523. * the alloc/free process all the time. In a small machine, 4 kmem-limited
  524. * cgroups is a reasonable guess. In the future, it could be a parameter or
  525. * tunable, but that is strictly not necessary.
  526. *
  527. * MAX_SIZE should be as large as the number of css_ids. Ideally, we could get
  528. * this constant directly from cgroup, but it is understandable that this is
  529. * better kept as an internal representation in cgroup.c. In any case, the
  530. * css_id space is not getting any smaller, and we don't have to necessarily
  531. * increase ours as well if it increases.
  532. */
  533. #define MEMCG_CACHES_MIN_SIZE 4
  534. #define MEMCG_CACHES_MAX_SIZE 65535
  535. /*
  536. * A lot of the calls to the cache allocation functions are expected to be
  537. * inlined by the compiler. Since the calls to memcg_kmem_get_cache are
  538. * conditional to this static branch, we'll have to allow modules that does
  539. * kmem_cache_alloc and the such to see this symbol as well
  540. */
  541. struct static_key memcg_kmem_enabled_key;
  542. EXPORT_SYMBOL(memcg_kmem_enabled_key);
  543. static void disarm_kmem_keys(struct mem_cgroup *memcg)
  544. {
  545. if (memcg_kmem_is_active(memcg)) {
  546. static_key_slow_dec(&memcg_kmem_enabled_key);
  547. ida_simple_remove(&kmem_limited_groups, memcg->kmemcg_id);
  548. }
  549. /*
  550. * This check can't live in kmem destruction function,
  551. * since the charges will outlive the cgroup
  552. */
  553. WARN_ON(res_counter_read_u64(&memcg->kmem, RES_USAGE) != 0);
  554. }
  555. #else
  556. static void disarm_kmem_keys(struct mem_cgroup *memcg)
  557. {
  558. }
  559. #endif /* CONFIG_MEMCG_KMEM */
  560. static void disarm_static_keys(struct mem_cgroup *memcg)
  561. {
  562. disarm_sock_keys(memcg);
  563. disarm_kmem_keys(memcg);
  564. }
  565. static void drain_all_stock_async(struct mem_cgroup *memcg);
  566. static struct mem_cgroup_per_zone *
  567. mem_cgroup_zoneinfo(struct mem_cgroup *memcg, int nid, int zid)
  568. {
  569. VM_BUG_ON((unsigned)nid >= nr_node_ids);
  570. return &memcg->info.nodeinfo[nid]->zoneinfo[zid];
  571. }
  572. struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg)
  573. {
  574. return &memcg->css;
  575. }
  576. static struct mem_cgroup_per_zone *
  577. page_cgroup_zoneinfo(struct mem_cgroup *memcg, struct page *page)
  578. {
  579. int nid = page_to_nid(page);
  580. int zid = page_zonenum(page);
  581. return mem_cgroup_zoneinfo(memcg, nid, zid);
  582. }
  583. static struct mem_cgroup_tree_per_zone *
  584. soft_limit_tree_node_zone(int nid, int zid)
  585. {
  586. return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
  587. }
  588. static struct mem_cgroup_tree_per_zone *
  589. soft_limit_tree_from_page(struct page *page)
  590. {
  591. int nid = page_to_nid(page);
  592. int zid = page_zonenum(page);
  593. return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
  594. }
  595. static void
  596. __mem_cgroup_insert_exceeded(struct mem_cgroup *memcg,
  597. struct mem_cgroup_per_zone *mz,
  598. struct mem_cgroup_tree_per_zone *mctz,
  599. unsigned long long new_usage_in_excess)
  600. {
  601. struct rb_node **p = &mctz->rb_root.rb_node;
  602. struct rb_node *parent = NULL;
  603. struct mem_cgroup_per_zone *mz_node;
  604. if (mz->on_tree)
  605. return;
  606. mz->usage_in_excess = new_usage_in_excess;
  607. if (!mz->usage_in_excess)
  608. return;
  609. while (*p) {
  610. parent = *p;
  611. mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
  612. tree_node);
  613. if (mz->usage_in_excess < mz_node->usage_in_excess)
  614. p = &(*p)->rb_left;
  615. /*
  616. * We can't avoid mem cgroups that are over their soft
  617. * limit by the same amount
  618. */
  619. else if (mz->usage_in_excess >= mz_node->usage_in_excess)
  620. p = &(*p)->rb_right;
  621. }
  622. rb_link_node(&mz->tree_node, parent, p);
  623. rb_insert_color(&mz->tree_node, &mctz->rb_root);
  624. mz->on_tree = true;
  625. }
  626. static void
  627. __mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
  628. struct mem_cgroup_per_zone *mz,
  629. struct mem_cgroup_tree_per_zone *mctz)
  630. {
  631. if (!mz->on_tree)
  632. return;
  633. rb_erase(&mz->tree_node, &mctz->rb_root);
  634. mz->on_tree = false;
  635. }
  636. static void
  637. mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
  638. struct mem_cgroup_per_zone *mz,
  639. struct mem_cgroup_tree_per_zone *mctz)
  640. {
  641. spin_lock(&mctz->lock);
  642. __mem_cgroup_remove_exceeded(memcg, mz, mctz);
  643. spin_unlock(&mctz->lock);
  644. }
  645. static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
  646. {
  647. unsigned long long excess;
  648. struct mem_cgroup_per_zone *mz;
  649. struct mem_cgroup_tree_per_zone *mctz;
  650. int nid = page_to_nid(page);
  651. int zid = page_zonenum(page);
  652. mctz = soft_limit_tree_from_page(page);
  653. /*
  654. * Necessary to update all ancestors when hierarchy is used.
  655. * because their event counter is not touched.
  656. */
  657. for (; memcg; memcg = parent_mem_cgroup(memcg)) {
  658. mz = mem_cgroup_zoneinfo(memcg, nid, zid);
  659. excess = res_counter_soft_limit_excess(&memcg->res);
  660. /*
  661. * We have to update the tree if mz is on RB-tree or
  662. * mem is over its softlimit.
  663. */
  664. if (excess || mz->on_tree) {
  665. spin_lock(&mctz->lock);
  666. /* if on-tree, remove it */
  667. if (mz->on_tree)
  668. __mem_cgroup_remove_exceeded(memcg, mz, mctz);
  669. /*
  670. * Insert again. mz->usage_in_excess will be updated.
  671. * If excess is 0, no tree ops.
  672. */
  673. __mem_cgroup_insert_exceeded(memcg, mz, mctz, excess);
  674. spin_unlock(&mctz->lock);
  675. }
  676. }
  677. }
  678. static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
  679. {
  680. int node, zone;
  681. struct mem_cgroup_per_zone *mz;
  682. struct mem_cgroup_tree_per_zone *mctz;
  683. for_each_node(node) {
  684. for (zone = 0; zone < MAX_NR_ZONES; zone++) {
  685. mz = mem_cgroup_zoneinfo(memcg, node, zone);
  686. mctz = soft_limit_tree_node_zone(node, zone);
  687. mem_cgroup_remove_exceeded(memcg, mz, mctz);
  688. }
  689. }
  690. }
  691. static struct mem_cgroup_per_zone *
  692. __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
  693. {
  694. struct rb_node *rightmost = NULL;
  695. struct mem_cgroup_per_zone *mz;
  696. retry:
  697. mz = NULL;
  698. rightmost = rb_last(&mctz->rb_root);
  699. if (!rightmost)
  700. goto done; /* Nothing to reclaim from */
  701. mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
  702. /*
  703. * Remove the node now but someone else can add it back,
  704. * we will to add it back at the end of reclaim to its correct
  705. * position in the tree.
  706. */
  707. __mem_cgroup_remove_exceeded(mz->memcg, mz, mctz);
  708. if (!res_counter_soft_limit_excess(&mz->memcg->res) ||
  709. !css_tryget(&mz->memcg->css))
  710. goto retry;
  711. done:
  712. return mz;
  713. }
  714. static struct mem_cgroup_per_zone *
  715. mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
  716. {
  717. struct mem_cgroup_per_zone *mz;
  718. spin_lock(&mctz->lock);
  719. mz = __mem_cgroup_largest_soft_limit_node(mctz);
  720. spin_unlock(&mctz->lock);
  721. return mz;
  722. }
  723. /*
  724. * Implementation Note: reading percpu statistics for memcg.
  725. *
  726. * Both of vmstat[] and percpu_counter has threshold and do periodic
  727. * synchronization to implement "quick" read. There are trade-off between
  728. * reading cost and precision of value. Then, we may have a chance to implement
  729. * a periodic synchronizion of counter in memcg's counter.
  730. *
  731. * But this _read() function is used for user interface now. The user accounts
  732. * memory usage by memory cgroup and he _always_ requires exact value because
  733. * he accounts memory. Even if we provide quick-and-fuzzy read, we always
  734. * have to visit all online cpus and make sum. So, for now, unnecessary
  735. * synchronization is not implemented. (just implemented for cpu hotplug)
  736. *
  737. * If there are kernel internal actions which can make use of some not-exact
  738. * value, and reading all cpu value can be performance bottleneck in some
  739. * common workload, threashold and synchonization as vmstat[] should be
  740. * implemented.
  741. */
  742. static long mem_cgroup_read_stat(struct mem_cgroup *memcg,
  743. enum mem_cgroup_stat_index idx)
  744. {
  745. long val = 0;
  746. int cpu;
  747. get_online_cpus();
  748. for_each_online_cpu(cpu)
  749. val += per_cpu(memcg->stat->count[idx], cpu);
  750. #ifdef CONFIG_HOTPLUG_CPU
  751. spin_lock(&memcg->pcp_counter_lock);
  752. val += memcg->nocpu_base.count[idx];
  753. spin_unlock(&memcg->pcp_counter_lock);
  754. #endif
  755. put_online_cpus();
  756. return val;
  757. }
  758. static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,
  759. bool charge)
  760. {
  761. int val = (charge) ? 1 : -1;
  762. this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SWAP], val);
  763. }
  764. static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg,
  765. enum mem_cgroup_events_index idx)
  766. {
  767. unsigned long val = 0;
  768. int cpu;
  769. for_each_online_cpu(cpu)
  770. val += per_cpu(memcg->stat->events[idx], cpu);
  771. #ifdef CONFIG_HOTPLUG_CPU
  772. spin_lock(&memcg->pcp_counter_lock);
  773. val += memcg->nocpu_base.events[idx];
  774. spin_unlock(&memcg->pcp_counter_lock);
  775. #endif
  776. return val;
  777. }
  778. static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
  779. bool anon, int nr_pages)
  780. {
  781. preempt_disable();
  782. /*
  783. * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
  784. * counted as CACHE even if it's on ANON LRU.
  785. */
  786. if (anon)
  787. __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS],
  788. nr_pages);
  789. else
  790. __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE],
  791. nr_pages);
  792. /* pagein of a big page is an event. So, ignore page size */
  793. if (nr_pages > 0)
  794. __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGIN]);
  795. else {
  796. __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]);
  797. nr_pages = -nr_pages; /* for event */
  798. }
  799. __this_cpu_add(memcg->stat->nr_page_events, nr_pages);
  800. preempt_enable();
  801. }
  802. unsigned long
  803. mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
  804. {
  805. struct mem_cgroup_per_zone *mz;
  806. mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
  807. return mz->lru_size[lru];
  808. }
  809. static unsigned long
  810. mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,
  811. unsigned int lru_mask)
  812. {
  813. struct mem_cgroup_per_zone *mz;
  814. enum lru_list lru;
  815. unsigned long ret = 0;
  816. mz = mem_cgroup_zoneinfo(memcg, nid, zid);
  817. for_each_lru(lru) {
  818. if (BIT(lru) & lru_mask)
  819. ret += mz->lru_size[lru];
  820. }
  821. return ret;
  822. }
  823. static unsigned long
  824. mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
  825. int nid, unsigned int lru_mask)
  826. {
  827. u64 total = 0;
  828. int zid;
  829. for (zid = 0; zid < MAX_NR_ZONES; zid++)
  830. total += mem_cgroup_zone_nr_lru_pages(memcg,
  831. nid, zid, lru_mask);
  832. return total;
  833. }
  834. static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
  835. unsigned int lru_mask)
  836. {
  837. int nid;
  838. u64 total = 0;
  839. for_each_node_state(nid, N_MEMORY)
  840. total += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
  841. return total;
  842. }
  843. static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
  844. enum mem_cgroup_events_target target)
  845. {
  846. unsigned long val, next;
  847. val = __this_cpu_read(memcg->stat->nr_page_events);
  848. next = __this_cpu_read(memcg->stat->targets[target]);
  849. /* from time_after() in jiffies.h */
  850. if ((long)next - (long)val < 0) {
  851. switch (target) {
  852. case MEM_CGROUP_TARGET_THRESH:
  853. next = val + THRESHOLDS_EVENTS_TARGET;
  854. break;
  855. case MEM_CGROUP_TARGET_SOFTLIMIT:
  856. next = val + SOFTLIMIT_EVENTS_TARGET;
  857. break;
  858. case MEM_CGROUP_TARGET_NUMAINFO:
  859. next = val + NUMAINFO_EVENTS_TARGET;
  860. break;
  861. default:
  862. break;
  863. }
  864. __this_cpu_write(memcg->stat->targets[target], next);
  865. return true;
  866. }
  867. return false;
  868. }
  869. /*
  870. * Check events in order.
  871. *
  872. */
  873. static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
  874. {
  875. preempt_disable();
  876. /* threshold event is triggered in finer grain than soft limit */
  877. if (unlikely(mem_cgroup_event_ratelimit(memcg,
  878. MEM_CGROUP_TARGET_THRESH))) {
  879. bool do_softlimit;
  880. bool do_numainfo __maybe_unused;
  881. do_softlimit = mem_cgroup_event_ratelimit(memcg,
  882. MEM_CGROUP_TARGET_SOFTLIMIT);
  883. #if MAX_NUMNODES > 1
  884. do_numainfo = mem_cgroup_event_ratelimit(memcg,
  885. MEM_CGROUP_TARGET_NUMAINFO);
  886. #endif
  887. preempt_enable();
  888. mem_cgroup_threshold(memcg);
  889. if (unlikely(do_softlimit))
  890. mem_cgroup_update_tree(memcg, page);
  891. #if MAX_NUMNODES > 1
  892. if (unlikely(do_numainfo))
  893. atomic_inc(&memcg->numainfo_events);
  894. #endif
  895. } else
  896. preempt_enable();
  897. }
  898. struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
  899. {
  900. return mem_cgroup_from_css(
  901. cgroup_subsys_state(cont, mem_cgroup_subsys_id));
  902. }
  903. struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
  904. {
  905. /*
  906. * mm_update_next_owner() may clear mm->owner to NULL
  907. * if it races with swapoff, page migration, etc.
  908. * So this can be called with p == NULL.
  909. */
  910. if (unlikely(!p))
  911. return NULL;
  912. return mem_cgroup_from_css(task_subsys_state(p, mem_cgroup_subsys_id));
  913. }
  914. struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
  915. {
  916. struct mem_cgroup *memcg = NULL;
  917. if (!mm)
  918. return NULL;
  919. /*
  920. * Because we have no locks, mm->owner's may be being moved to other
  921. * cgroup. We use css_tryget() here even if this looks
  922. * pessimistic (rather than adding locks here).
  923. */
  924. rcu_read_lock();
  925. do {
  926. memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
  927. if (unlikely(!memcg))
  928. break;
  929. } while (!css_tryget(&memcg->css));
  930. rcu_read_unlock();
  931. return memcg;
  932. }
  933. /**
  934. * mem_cgroup_iter - iterate over memory cgroup hierarchy
  935. * @root: hierarchy root
  936. * @prev: previously returned memcg, NULL on first invocation
  937. * @reclaim: cookie for shared reclaim walks, NULL for full walks
  938. *
  939. * Returns references to children of the hierarchy below @root, or
  940. * @root itself, or %NULL after a full round-trip.
  941. *
  942. * Caller must pass the return value in @prev on subsequent
  943. * invocations for reference counting, or use mem_cgroup_iter_break()
  944. * to cancel a hierarchy walk before the round-trip is complete.
  945. *
  946. * Reclaimers can specify a zone and a priority level in @reclaim to
  947. * divide up the memcgs in the hierarchy among all concurrent
  948. * reclaimers operating on the same zone and priority.
  949. */
  950. struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
  951. struct mem_cgroup *prev,
  952. struct mem_cgroup_reclaim_cookie *reclaim)
  953. {
  954. struct mem_cgroup *memcg = NULL;
  955. int id = 0;
  956. if (mem_cgroup_disabled())
  957. return NULL;
  958. if (!root)
  959. root = root_mem_cgroup;
  960. if (prev && !reclaim)
  961. id = css_id(&prev->css);
  962. if (!root->use_hierarchy && root != root_mem_cgroup) {
  963. if (prev)
  964. goto out_css_put;
  965. return root;
  966. }
  967. while (!memcg) {
  968. struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
  969. struct cgroup_subsys_state *css;
  970. if (reclaim) {
  971. int nid = zone_to_nid(reclaim->zone);
  972. int zid = zone_idx(reclaim->zone);
  973. struct mem_cgroup_per_zone *mz;
  974. mz = mem_cgroup_zoneinfo(root, nid, zid);
  975. iter = &mz->reclaim_iter[reclaim->priority];
  976. if (prev && reclaim->generation != iter->generation)
  977. goto out_css_put;
  978. id = iter->position;
  979. }
  980. rcu_read_lock();
  981. css = css_get_next(&mem_cgroup_subsys, id + 1, &root->css, &id);
  982. if (css) {
  983. if (css == &root->css || css_tryget(css))
  984. memcg = mem_cgroup_from_css(css);
  985. } else
  986. id = 0;
  987. rcu_read_unlock();
  988. if (reclaim) {
  989. iter->position = id;
  990. if (!css)
  991. iter->generation++;
  992. else if (!prev && memcg)
  993. reclaim->generation = iter->generation;
  994. }
  995. if (prev && !css)
  996. goto out_css_put;
  997. }
  998. out_css_put:
  999. if (prev && prev != root)
  1000. css_put(&prev->css);
  1001. return memcg;
  1002. }
  1003. /**
  1004. * mem_cgroup_iter_break - abort a hierarchy walk prematurely
  1005. * @root: hierarchy root
  1006. * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
  1007. */
  1008. void mem_cgroup_iter_break(struct mem_cgroup *root,
  1009. struct mem_cgroup *prev)
  1010. {
  1011. if (!root)
  1012. root = root_mem_cgroup;
  1013. if (prev && prev != root)
  1014. css_put(&prev->css);
  1015. }
  1016. /*
  1017. * Iteration constructs for visiting all cgroups (under a tree). If
  1018. * loops are exited prematurely (break), mem_cgroup_iter_break() must
  1019. * be used for reference counting.
  1020. */
  1021. #define for_each_mem_cgroup_tree(iter, root) \
  1022. for (iter = mem_cgroup_iter(root, NULL, NULL); \
  1023. iter != NULL; \
  1024. iter = mem_cgroup_iter(root, iter, NULL))
  1025. #define for_each_mem_cgroup(iter) \
  1026. for (iter = mem_cgroup_iter(NULL, NULL, NULL); \
  1027. iter != NULL; \
  1028. iter = mem_cgroup_iter(NULL, iter, NULL))
  1029. void __mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx)
  1030. {
  1031. struct mem_cgroup *memcg;
  1032. rcu_read_lock();
  1033. memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
  1034. if (unlikely(!memcg))
  1035. goto out;
  1036. switch (idx) {
  1037. case PGFAULT:
  1038. this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGFAULT]);
  1039. break;
  1040. case PGMAJFAULT:
  1041. this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
  1042. break;
  1043. default:
  1044. BUG();
  1045. }
  1046. out:
  1047. rcu_read_unlock();
  1048. }
  1049. EXPORT_SYMBOL(__mem_cgroup_count_vm_event);
  1050. /**
  1051. * mem_cgroup_zone_lruvec - get the lru list vector for a zone and memcg
  1052. * @zone: zone of the wanted lruvec
  1053. * @memcg: memcg of the wanted lruvec
  1054. *
  1055. * Returns the lru list vector holding pages for the given @zone and
  1056. * @mem. This can be the global zone lruvec, if the memory controller
  1057. * is disabled.
  1058. */
  1059. struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
  1060. struct mem_cgroup *memcg)
  1061. {
  1062. struct mem_cgroup_per_zone *mz;
  1063. struct lruvec *lruvec;
  1064. if (mem_cgroup_disabled()) {
  1065. lruvec = &zone->lruvec;
  1066. goto out;
  1067. }
  1068. mz = mem_cgroup_zoneinfo(memcg, zone_to_nid(zone), zone_idx(zone));
  1069. lruvec = &mz->lruvec;
  1070. out:
  1071. /*
  1072. * Since a node can be onlined after the mem_cgroup was created,
  1073. * we have to be prepared to initialize lruvec->zone here;
  1074. * and if offlined then reonlined, we need to reinitialize it.
  1075. */
  1076. if (unlikely(lruvec->zone != zone))
  1077. lruvec->zone = zone;
  1078. return lruvec;
  1079. }
  1080. /*
  1081. * Following LRU functions are allowed to be used without PCG_LOCK.
  1082. * Operations are called by routine of global LRU independently from memcg.
  1083. * What we have to take care of here is validness of pc->mem_cgroup.
  1084. *
  1085. * Changes to pc->mem_cgroup happens when
  1086. * 1. charge
  1087. * 2. moving account
  1088. * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
  1089. * It is added to LRU before charge.
  1090. * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
  1091. * When moving account, the page is not on LRU. It's isolated.
  1092. */
  1093. /**
  1094. * mem_cgroup_page_lruvec - return lruvec for adding an lru page
  1095. * @page: the page
  1096. * @zone: zone of the page
  1097. */
  1098. struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct zone *zone)
  1099. {
  1100. struct mem_cgroup_per_zone *mz;
  1101. struct mem_cgroup *memcg;
  1102. struct page_cgroup *pc;
  1103. struct lruvec *lruvec;
  1104. if (mem_cgroup_disabled()) {
  1105. lruvec = &zone->lruvec;
  1106. goto out;
  1107. }
  1108. pc = lookup_page_cgroup(page);
  1109. memcg = pc->mem_cgroup;
  1110. /*
  1111. * Surreptitiously switch any uncharged offlist page to root:
  1112. * an uncharged page off lru does nothing to secure
  1113. * its former mem_cgroup from sudden removal.
  1114. *
  1115. * Our caller holds lru_lock, and PageCgroupUsed is updated
  1116. * under page_cgroup lock: between them, they make all uses
  1117. * of pc->mem_cgroup safe.
  1118. */
  1119. if (!PageLRU(page) && !PageCgroupUsed(pc) && memcg != root_mem_cgroup)
  1120. pc->mem_cgroup = memcg = root_mem_cgroup;
  1121. mz = page_cgroup_zoneinfo(memcg, page);
  1122. lruvec = &mz->lruvec;
  1123. out:
  1124. /*
  1125. * Since a node can be onlined after the mem_cgroup was created,
  1126. * we have to be prepared to initialize lruvec->zone here;
  1127. * and if offlined then reonlined, we need to reinitialize it.
  1128. */
  1129. if (unlikely(lruvec->zone != zone))
  1130. lruvec->zone = zone;
  1131. return lruvec;
  1132. }
  1133. /**
  1134. * mem_cgroup_update_lru_size - account for adding or removing an lru page
  1135. * @lruvec: mem_cgroup per zone lru vector
  1136. * @lru: index of lru list the page is sitting on
  1137. * @nr_pages: positive when adding or negative when removing
  1138. *
  1139. * This function must be called when a page is added to or removed from an
  1140. * lru list.
  1141. */
  1142. void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
  1143. int nr_pages)
  1144. {
  1145. struct mem_cgroup_per_zone *mz;
  1146. unsigned long *lru_size;
  1147. if (mem_cgroup_disabled())
  1148. return;
  1149. mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
  1150. lru_size = mz->lru_size + lru;
  1151. *lru_size += nr_pages;
  1152. VM_BUG_ON((long)(*lru_size) < 0);
  1153. }
  1154. /*
  1155. * Checks whether given mem is same or in the root_mem_cgroup's
  1156. * hierarchy subtree
  1157. */
  1158. bool __mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
  1159. struct mem_cgroup *memcg)
  1160. {
  1161. if (root_memcg == memcg)
  1162. return true;
  1163. if (!root_memcg->use_hierarchy || !memcg)
  1164. return false;
  1165. return css_is_ancestor(&memcg->css, &root_memcg->css);
  1166. }
  1167. static bool mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
  1168. struct mem_cgroup *memcg)
  1169. {
  1170. bool ret;
  1171. rcu_read_lock();
  1172. ret = __mem_cgroup_same_or_subtree(root_memcg, memcg);
  1173. rcu_read_unlock();
  1174. return ret;
  1175. }
  1176. int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *memcg)
  1177. {
  1178. int ret;
  1179. struct mem_cgroup *curr = NULL;
  1180. struct task_struct *p;
  1181. p = find_lock_task_mm(task);
  1182. if (p) {
  1183. curr = try_get_mem_cgroup_from_mm(p->mm);
  1184. task_unlock(p);
  1185. } else {
  1186. /*
  1187. * All threads may have already detached their mm's, but the oom
  1188. * killer still needs to detect if they have already been oom
  1189. * killed to prevent needlessly killing additional tasks.
  1190. */
  1191. task_lock(task);
  1192. curr = mem_cgroup_from_task(task);
  1193. if (curr)
  1194. css_get(&curr->css);
  1195. task_unlock(task);
  1196. }
  1197. if (!curr)
  1198. return 0;
  1199. /*
  1200. * We should check use_hierarchy of "memcg" not "curr". Because checking
  1201. * use_hierarchy of "curr" here make this function true if hierarchy is
  1202. * enabled in "curr" and "curr" is a child of "memcg" in *cgroup*
  1203. * hierarchy(even if use_hierarchy is disabled in "memcg").
  1204. */
  1205. ret = mem_cgroup_same_or_subtree(memcg, curr);
  1206. css_put(&curr->css);
  1207. return ret;
  1208. }
  1209. int mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec)
  1210. {
  1211. unsigned long inactive_ratio;
  1212. unsigned long inactive;
  1213. unsigned long active;
  1214. unsigned long gb;
  1215. inactive = mem_cgroup_get_lru_size(lruvec, LRU_INACTIVE_ANON);
  1216. active = mem_cgroup_get_lru_size(lruvec, LRU_ACTIVE_ANON);
  1217. gb = (inactive + active) >> (30 - PAGE_SHIFT);
  1218. if (gb)
  1219. inactive_ratio = int_sqrt(10 * gb);
  1220. else
  1221. inactive_ratio = 1;
  1222. return inactive * inactive_ratio < active;
  1223. }
  1224. #define mem_cgroup_from_res_counter(counter, member) \
  1225. container_of(counter, struct mem_cgroup, member)
  1226. /**
  1227. * mem_cgroup_margin - calculate chargeable space of a memory cgroup
  1228. * @memcg: the memory cgroup
  1229. *
  1230. * Returns the maximum amount of memory @mem can be charged with, in
  1231. * pages.
  1232. */
  1233. static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
  1234. {
  1235. unsigned long long margin;
  1236. margin = res_counter_margin(&memcg->res);
  1237. if (do_swap_account)
  1238. margin = min(margin, res_counter_margin(&memcg->memsw));
  1239. return margin >> PAGE_SHIFT;
  1240. }
  1241. int mem_cgroup_swappiness(struct mem_cgroup *memcg)
  1242. {
  1243. struct cgroup *cgrp = memcg->css.cgroup;
  1244. /* root ? */
  1245. if (cgrp->parent == NULL)
  1246. return vm_swappiness;
  1247. return memcg->swappiness;
  1248. }
  1249. /*
  1250. * memcg->moving_account is used for checking possibility that some thread is
  1251. * calling move_account(). When a thread on CPU-A starts moving pages under
  1252. * a memcg, other threads should check memcg->moving_account under
  1253. * rcu_read_lock(), like this:
  1254. *
  1255. * CPU-A CPU-B
  1256. * rcu_read_lock()
  1257. * memcg->moving_account+1 if (memcg->mocing_account)
  1258. * take heavy locks.
  1259. * synchronize_rcu() update something.
  1260. * rcu_read_unlock()
  1261. * start move here.
  1262. */
  1263. /* for quick checking without looking up memcg */
  1264. atomic_t memcg_moving __read_mostly;
  1265. static void mem_cgroup_start_move(struct mem_cgroup *memcg)
  1266. {
  1267. atomic_inc(&memcg_moving);
  1268. atomic_inc(&memcg->moving_account);
  1269. synchronize_rcu();
  1270. }
  1271. static void mem_cgroup_end_move(struct mem_cgroup *memcg)
  1272. {
  1273. /*
  1274. * Now, mem_cgroup_clear_mc() may call this function with NULL.
  1275. * We check NULL in callee rather than caller.
  1276. */
  1277. if (memcg) {
  1278. atomic_dec(&memcg_moving);
  1279. atomic_dec(&memcg->moving_account);
  1280. }
  1281. }
  1282. /*
  1283. * 2 routines for checking "mem" is under move_account() or not.
  1284. *
  1285. * mem_cgroup_stolen() - checking whether a cgroup is mc.from or not. This
  1286. * is used for avoiding races in accounting. If true,
  1287. * pc->mem_cgroup may be overwritten.
  1288. *
  1289. * mem_cgroup_under_move() - checking a cgroup is mc.from or mc.to or
  1290. * under hierarchy of moving cgroups. This is for
  1291. * waiting at hith-memory prressure caused by "move".
  1292. */
  1293. static bool mem_cgroup_stolen(struct mem_cgroup *memcg)
  1294. {
  1295. VM_BUG_ON(!rcu_read_lock_held());
  1296. return atomic_read(&memcg->moving_account) > 0;
  1297. }
  1298. static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
  1299. {
  1300. struct mem_cgroup *from;
  1301. struct mem_cgroup *to;
  1302. bool ret = false;
  1303. /*
  1304. * Unlike task_move routines, we access mc.to, mc.from not under
  1305. * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
  1306. */
  1307. spin_lock(&mc.lock);
  1308. from = mc.from;
  1309. to = mc.to;
  1310. if (!from)
  1311. goto unlock;
  1312. ret = mem_cgroup_same_or_subtree(memcg, from)
  1313. || mem_cgroup_same_or_subtree(memcg, to);
  1314. unlock:
  1315. spin_unlock(&mc.lock);
  1316. return ret;
  1317. }
  1318. static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
  1319. {
  1320. if (mc.moving_task && current != mc.moving_task) {
  1321. if (mem_cgroup_under_move(memcg)) {
  1322. DEFINE_WAIT(wait);
  1323. prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
  1324. /* moving charge context might have finished. */
  1325. if (mc.moving_task)
  1326. schedule();
  1327. finish_wait(&mc.waitq, &wait);
  1328. return true;
  1329. }
  1330. }
  1331. return false;
  1332. }
  1333. /*
  1334. * Take this lock when
  1335. * - a code tries to modify page's memcg while it's USED.
  1336. * - a code tries to modify page state accounting in a memcg.
  1337. * see mem_cgroup_stolen(), too.
  1338. */
  1339. static void move_lock_mem_cgroup(struct mem_cgroup *memcg,
  1340. unsigned long *flags)
  1341. {
  1342. spin_lock_irqsave(&memcg->move_lock, *flags);
  1343. }
  1344. static void move_unlock_mem_cgroup(struct mem_cgroup *memcg,
  1345. unsigned long *flags)
  1346. {
  1347. spin_unlock_irqrestore(&memcg->move_lock, *flags);
  1348. }
  1349. #define K(x) ((x) << (PAGE_SHIFT-10))
  1350. /**
  1351. * mem_cgroup_print_oom_info: Print OOM information relevant to memory controller.
  1352. * @memcg: The memory cgroup that went over limit
  1353. * @p: Task that is going to be killed
  1354. *
  1355. * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
  1356. * enabled
  1357. */
  1358. void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
  1359. {
  1360. struct cgroup *task_cgrp;
  1361. struct cgroup *mem_cgrp;
  1362. /*
  1363. * Need a buffer in BSS, can't rely on allocations. The code relies
  1364. * on the assumption that OOM is serialized for memory controller.
  1365. * If this assumption is broken, revisit this code.
  1366. */
  1367. static char memcg_name[PATH_MAX];
  1368. int ret;
  1369. struct mem_cgroup *iter;
  1370. unsigned int i;
  1371. if (!p)
  1372. return;
  1373. rcu_read_lock();
  1374. mem_cgrp = memcg->css.cgroup;
  1375. task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);
  1376. ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
  1377. if (ret < 0) {
  1378. /*
  1379. * Unfortunately, we are unable to convert to a useful name
  1380. * But we'll still print out the usage information
  1381. */
  1382. rcu_read_unlock();
  1383. goto done;
  1384. }
  1385. rcu_read_unlock();
  1386. pr_info("Task in %s killed", memcg_name);
  1387. rcu_read_lock();
  1388. ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
  1389. if (ret < 0) {
  1390. rcu_read_unlock();
  1391. goto done;
  1392. }
  1393. rcu_read_unlock();
  1394. /*
  1395. * Continues from above, so we don't need an KERN_ level
  1396. */
  1397. pr_cont(" as a result of limit of %s\n", memcg_name);
  1398. done:
  1399. pr_info("memory: usage %llukB, limit %llukB, failcnt %llu\n",
  1400. res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
  1401. res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
  1402. res_counter_read_u64(&memcg->res, RES_FAILCNT));
  1403. pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %llu\n",
  1404. res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
  1405. res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
  1406. res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
  1407. pr_info("kmem: usage %llukB, limit %llukB, failcnt %llu\n",
  1408. res_counter_read_u64(&memcg->kmem, RES_USAGE) >> 10,
  1409. res_counter_read_u64(&memcg->kmem, RES_LIMIT) >> 10,
  1410. res_counter_read_u64(&memcg->kmem, RES_FAILCNT));
  1411. for_each_mem_cgroup_tree(iter, memcg) {
  1412. pr_info("Memory cgroup stats");
  1413. rcu_read_lock();
  1414. ret = cgroup_path(iter->css.cgroup, memcg_name, PATH_MAX);
  1415. if (!ret)
  1416. pr_cont(" for %s", memcg_name);
  1417. rcu_read_unlock();
  1418. pr_cont(":");
  1419. for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
  1420. if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
  1421. continue;
  1422. pr_cont(" %s:%ldKB", mem_cgroup_stat_names[i],
  1423. K(mem_cgroup_read_stat(iter, i)));
  1424. }
  1425. for (i = 0; i < NR_LRU_LISTS; i++)
  1426. pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
  1427. K(mem_cgroup_nr_lru_pages(iter, BIT(i))));
  1428. pr_cont("\n");
  1429. }
  1430. }
  1431. /*
  1432. * This function returns the number of memcg under hierarchy tree. Returns
  1433. * 1(self count) if no children.
  1434. */
  1435. static int mem_cgroup_count_children(struct mem_cgroup *memcg)
  1436. {
  1437. int num = 0;
  1438. struct mem_cgroup *iter;
  1439. for_each_mem_cgroup_tree(iter, memcg)
  1440. num++;
  1441. return num;
  1442. }
  1443. /*
  1444. * Return the memory (and swap, if configured) limit for a memcg.
  1445. */
  1446. static u64 mem_cgroup_get_limit(struct mem_cgroup *memcg)
  1447. {
  1448. u64 limit;
  1449. limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
  1450. /*
  1451. * Do not consider swap space if we cannot swap due to swappiness
  1452. */
  1453. if (mem_cgroup_swappiness(memcg)) {
  1454. u64 memsw;
  1455. limit += total_swap_pages << PAGE_SHIFT;
  1456. memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
  1457. /*
  1458. * If memsw is finite and limits the amount of swap space
  1459. * available to this memcg, return that limit.
  1460. */
  1461. limit = min(limit, memsw);
  1462. }
  1463. return limit;
  1464. }
  1465. static void mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
  1466. int order)
  1467. {
  1468. struct mem_cgroup *iter;
  1469. unsigned long chosen_points = 0;
  1470. unsigned long totalpages;
  1471. unsigned int points = 0;
  1472. struct task_struct *chosen = NULL;
  1473. /*
  1474. * If current has a pending SIGKILL, then automatically select it. The
  1475. * goal is to allow it to allocate so that it may quickly exit and free
  1476. * its memory.
  1477. */
  1478. if (fatal_signal_pending(current)) {
  1479. set_thread_flag(TIF_MEMDIE);
  1480. return;
  1481. }
  1482. check_panic_on_oom(CONSTRAINT_MEMCG, gfp_mask, order, NULL);
  1483. totalpages = mem_cgroup_get_limit(memcg) >> PAGE_SHIFT ? : 1;
  1484. for_each_mem_cgroup_tree(iter, memcg) {
  1485. struct cgroup *cgroup = iter->css.cgroup;
  1486. struct cgroup_iter it;
  1487. struct task_struct *task;
  1488. cgroup_iter_start(cgroup, &it);
  1489. while ((task = cgroup_iter_next(cgroup, &it))) {
  1490. switch (oom_scan_process_thread(task, totalpages, NULL,
  1491. false)) {
  1492. case OOM_SCAN_SELECT:
  1493. if (chosen)
  1494. put_task_struct(chosen);
  1495. chosen = task;
  1496. chosen_points = ULONG_MAX;
  1497. get_task_struct(chosen);
  1498. /* fall through */
  1499. case OOM_SCAN_CONTINUE:
  1500. continue;
  1501. case OOM_SCAN_ABORT:
  1502. cgroup_iter_end(cgroup, &it);
  1503. mem_cgroup_iter_break(memcg, iter);
  1504. if (chosen)
  1505. put_task_struct(chosen);
  1506. return;
  1507. case OOM_SCAN_OK:
  1508. break;
  1509. };
  1510. points = oom_badness(task, memcg, NULL, totalpages);
  1511. if (points > chosen_points) {
  1512. if (chosen)
  1513. put_task_struct(chosen);
  1514. chosen = task;
  1515. chosen_points = points;
  1516. get_task_struct(chosen);
  1517. }
  1518. }
  1519. cgroup_iter_end(cgroup, &it);
  1520. }
  1521. if (!chosen)
  1522. return;
  1523. points = chosen_points * 1000 / totalpages;
  1524. oom_kill_process(chosen, gfp_mask, order, points, totalpages, memcg,
  1525. NULL, "Memory cgroup out of memory");
  1526. }
  1527. static unsigned long mem_cgroup_reclaim(struct mem_cgroup *memcg,
  1528. gfp_t gfp_mask,
  1529. unsigned long flags)
  1530. {
  1531. unsigned long total = 0;
  1532. bool noswap = false;
  1533. int loop;
  1534. if (flags & MEM_CGROUP_RECLAIM_NOSWAP)
  1535. noswap = true;
  1536. if (!(flags & MEM_CGROUP_RECLAIM_SHRINK) && memcg->memsw_is_minimum)
  1537. noswap = true;
  1538. for (loop = 0; loop < MEM_CGROUP_MAX_RECLAIM_LOOPS; loop++) {
  1539. if (loop)
  1540. drain_all_stock_async(memcg);
  1541. total += try_to_free_mem_cgroup_pages(memcg, gfp_mask, noswap);
  1542. /*
  1543. * Allow limit shrinkers, which are triggered directly
  1544. * by userspace, to catch signals and stop reclaim
  1545. * after minimal progress, regardless of the margin.
  1546. */
  1547. if (total && (flags & MEM_CGROUP_RECLAIM_SHRINK))
  1548. break;
  1549. if (mem_cgroup_margin(memcg))
  1550. break;
  1551. /*
  1552. * If nothing was reclaimed after two attempts, there
  1553. * may be no reclaimable pages in this hierarchy.
  1554. */
  1555. if (loop && !total)
  1556. break;
  1557. }
  1558. return total;
  1559. }
  1560. /**
  1561. * test_mem_cgroup_node_reclaimable
  1562. * @memcg: the target memcg
  1563. * @nid: the node ID to be checked.
  1564. * @noswap : specify true here if the user wants flle only information.
  1565. *
  1566. * This function returns whether the specified memcg contains any
  1567. * reclaimable pages on a node. Returns true if there are any reclaimable
  1568. * pages in the node.
  1569. */
  1570. static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
  1571. int nid, bool noswap)
  1572. {
  1573. if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
  1574. return true;
  1575. if (noswap || !total_swap_pages)
  1576. return false;
  1577. if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
  1578. return true;
  1579. return false;
  1580. }
  1581. #if MAX_NUMNODES > 1
  1582. /*
  1583. * Always updating the nodemask is not very good - even if we have an empty
  1584. * list or the wrong list here, we can start from some node and traverse all
  1585. * nodes based on the zonelist. So update the list loosely once per 10 secs.
  1586. *
  1587. */
  1588. static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
  1589. {
  1590. int nid;
  1591. /*
  1592. * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
  1593. * pagein/pageout changes since the last update.
  1594. */
  1595. if (!atomic_read(&memcg->numainfo_events))
  1596. return;
  1597. if (atomic_inc_return(&memcg->numainfo_updating) > 1)
  1598. return;
  1599. /* make a nodemask where this memcg uses memory from */
  1600. memcg->scan_nodes = node_states[N_MEMORY];
  1601. for_each_node_mask(nid, node_states[N_MEMORY]) {
  1602. if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
  1603. node_clear(nid, memcg->scan_nodes);
  1604. }
  1605. atomic_set(&memcg->numainfo_events, 0);
  1606. atomic_set(&memcg->numainfo_updating, 0);
  1607. }
  1608. /*
  1609. * Selecting a node where we start reclaim from. Because what we need is just
  1610. * reducing usage counter, start from anywhere is O,K. Considering
  1611. * memory reclaim from current node, there are pros. and cons.
  1612. *
  1613. * Freeing memory from current node means freeing memory from a node which
  1614. * we'll use or we've used. So, it may make LRU bad. And if several threads
  1615. * hit limits, it will see a contention on a node. But freeing from remote
  1616. * node means more costs for memory reclaim because of memory latency.
  1617. *
  1618. * Now, we use round-robin. Better algorithm is welcomed.
  1619. */
  1620. int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
  1621. {
  1622. int node;
  1623. mem_cgroup_may_update_nodemask(memcg);
  1624. node = memcg->last_scanned_node;
  1625. node = next_node(node, memcg->scan_nodes);
  1626. if (node == MAX_NUMNODES)
  1627. node = first_node(memcg->scan_nodes);
  1628. /*
  1629. * We call this when we hit limit, not when pages are added to LRU.
  1630. * No LRU may hold pages because all pages are UNEVICTABLE or
  1631. * memcg is too small and all pages are not on LRU. In that case,
  1632. * we use curret node.
  1633. */
  1634. if (unlikely(node == MAX_NUMNODES))
  1635. node = numa_node_id();
  1636. memcg->last_scanned_node = node;
  1637. return node;
  1638. }
  1639. /*
  1640. * Check all nodes whether it contains reclaimable pages or not.
  1641. * For quick scan, we make use of scan_nodes. This will allow us to skip
  1642. * unused nodes. But scan_nodes is lazily updated and may not cotain
  1643. * enough new information. We need to do double check.
  1644. */
  1645. static bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
  1646. {
  1647. int nid;
  1648. /*
  1649. * quick check...making use of scan_node.
  1650. * We can skip unused nodes.
  1651. */
  1652. if (!nodes_empty(memcg->scan_nodes)) {
  1653. for (nid = first_node(memcg->scan_nodes);
  1654. nid < MAX_NUMNODES;
  1655. nid = next_node(nid, memcg->scan_nodes)) {
  1656. if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
  1657. return true;
  1658. }
  1659. }
  1660. /*
  1661. * Check rest of nodes.
  1662. */
  1663. for_each_node_state(nid, N_MEMORY) {
  1664. if (node_isset(nid, memcg->scan_nodes))
  1665. continue;
  1666. if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
  1667. return true;
  1668. }
  1669. return false;
  1670. }
  1671. #else
  1672. int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
  1673. {
  1674. return 0;
  1675. }
  1676. static bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
  1677. {
  1678. return test_mem_cgroup_node_reclaimable(memcg, 0, noswap);
  1679. }
  1680. #endif
  1681. static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
  1682. struct zone *zone,
  1683. gfp_t gfp_mask,
  1684. unsigned long *total_scanned)
  1685. {
  1686. struct mem_cgroup *victim = NULL;
  1687. int total = 0;
  1688. int loop = 0;
  1689. unsigned long excess;
  1690. unsigned long nr_scanned;
  1691. struct mem_cgroup_reclaim_cookie reclaim = {
  1692. .zone = zone,
  1693. .priority = 0,
  1694. };
  1695. excess = res_counter_soft_limit_excess(&root_memcg->res) >> PAGE_SHIFT;
  1696. while (1) {
  1697. victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
  1698. if (!victim) {
  1699. loop++;
  1700. if (loop >= 2) {
  1701. /*
  1702. * If we have not been able to reclaim
  1703. * anything, it might because there are
  1704. * no reclaimable pages under this hierarchy
  1705. */
  1706. if (!total)
  1707. break;
  1708. /*
  1709. * We want to do more targeted reclaim.
  1710. * excess >> 2 is not to excessive so as to
  1711. * reclaim too much, nor too less that we keep
  1712. * coming back to reclaim from this cgroup
  1713. */
  1714. if (total >= (excess >> 2) ||
  1715. (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
  1716. break;
  1717. }
  1718. continue;
  1719. }
  1720. if (!mem_cgroup_reclaimable(victim, false))
  1721. continue;
  1722. total += mem_cgroup_shrink_node_zone(victim, gfp_mask, false,
  1723. zone, &nr_scanned);
  1724. *total_scanned += nr_scanned;
  1725. if (!res_counter_soft_limit_excess(&root_memcg->res))
  1726. break;
  1727. }
  1728. mem_cgroup_iter_break(root_memcg, victim);
  1729. return total;
  1730. }
  1731. /*
  1732. * Check OOM-Killer is already running under our hierarchy.
  1733. * If someone is running, return false.
  1734. * Has to be called with memcg_oom_lock
  1735. */
  1736. static bool mem_cgroup_oom_lock(struct mem_cgroup *memcg)
  1737. {
  1738. struct mem_cgroup *iter, *failed = NULL;
  1739. for_each_mem_cgroup_tree(iter, memcg) {
  1740. if (iter->oom_lock) {
  1741. /*
  1742. * this subtree of our hierarchy is already locked
  1743. * so we cannot give a lock.
  1744. */
  1745. failed = iter;
  1746. mem_cgroup_iter_break(memcg, iter);
  1747. break;
  1748. } else
  1749. iter->oom_lock = true;
  1750. }
  1751. if (!failed)
  1752. return true;
  1753. /*
  1754. * OK, we failed to lock the whole subtree so we have to clean up
  1755. * what we set up to the failing subtree
  1756. */
  1757. for_each_mem_cgroup_tree(iter, memcg) {
  1758. if (iter == failed) {
  1759. mem_cgroup_iter_break(memcg, iter);
  1760. break;
  1761. }
  1762. iter->oom_lock = false;
  1763. }
  1764. return false;
  1765. }
  1766. /*
  1767. * Has to be called with memcg_oom_lock
  1768. */
  1769. static int mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
  1770. {
  1771. struct mem_cgroup *iter;
  1772. for_each_mem_cgroup_tree(iter, memcg)
  1773. iter->oom_lock = false;
  1774. return 0;
  1775. }
  1776. static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
  1777. {
  1778. struct mem_cgroup *iter;
  1779. for_each_mem_cgroup_tree(iter, memcg)
  1780. atomic_inc(&iter->under_oom);
  1781. }
  1782. static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
  1783. {
  1784. struct mem_cgroup *iter;
  1785. /*
  1786. * When a new child is created while the hierarchy is under oom,
  1787. * mem_cgroup_oom_lock() may not be called. We have to use
  1788. * atomic_add_unless() here.
  1789. */
  1790. for_each_mem_cgroup_tree(iter, memcg)
  1791. atomic_add_unless(&iter->under_oom, -1, 0);
  1792. }
  1793. static DEFINE_SPINLOCK(memcg_oom_lock);
  1794. static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
  1795. struct oom_wait_info {
  1796. struct mem_cgroup *memcg;
  1797. wait_queue_t wait;
  1798. };
  1799. static int memcg_oom_wake_function(wait_queue_t *wait,
  1800. unsigned mode, int sync, void *arg)
  1801. {
  1802. struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
  1803. struct mem_cgroup *oom_wait_memcg;
  1804. struct oom_wait_info *oom_wait_info;
  1805. oom_wait_info = container_of(wait, struct oom_wait_info, wait);
  1806. oom_wait_memcg = oom_wait_info->memcg;
  1807. /*
  1808. * Both of oom_wait_info->memcg and wake_memcg are stable under us.
  1809. * Then we can use css_is_ancestor without taking care of RCU.
  1810. */
  1811. if (!mem_cgroup_same_or_subtree(oom_wait_memcg, wake_memcg)
  1812. && !mem_cgroup_same_or_subtree(wake_memcg, oom_wait_memcg))
  1813. return 0;
  1814. return autoremove_wake_function(wait, mode, sync, arg);
  1815. }
  1816. static void memcg_wakeup_oom(struct mem_cgroup *memcg)
  1817. {
  1818. /* for filtering, pass "memcg" as argument. */
  1819. __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
  1820. }
  1821. static void memcg_oom_recover(struct mem_cgroup *memcg)
  1822. {
  1823. if (memcg && atomic_read(&memcg->under_oom))
  1824. memcg_wakeup_oom(memcg);
  1825. }
  1826. /*
  1827. * try to call OOM killer. returns false if we should exit memory-reclaim loop.
  1828. */
  1829. static bool mem_cgroup_handle_oom(struct mem_cgroup *memcg, gfp_t mask,
  1830. int order)
  1831. {
  1832. struct oom_wait_info owait;
  1833. bool locked, need_to_kill;
  1834. owait.memcg = memcg;
  1835. owait.wait.flags = 0;
  1836. owait.wait.func = memcg_oom_wake_function;
  1837. owait.wait.private = current;
  1838. INIT_LIST_HEAD(&owait.wait.task_list);
  1839. need_to_kill = true;
  1840. mem_cgroup_mark_under_oom(memcg);
  1841. /* At first, try to OOM lock hierarchy under memcg.*/
  1842. spin_lock(&memcg_oom_lock);
  1843. locked = mem_cgroup_oom_lock(memcg);
  1844. /*
  1845. * Even if signal_pending(), we can't quit charge() loop without
  1846. * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL
  1847. * under OOM is always welcomed, use TASK_KILLABLE here.
  1848. */
  1849. prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
  1850. if (!locked || memcg->oom_kill_disable)
  1851. need_to_kill = false;
  1852. if (locked)
  1853. mem_cgroup_oom_notify(memcg);
  1854. spin_unlock(&memcg_oom_lock);
  1855. if (need_to_kill) {
  1856. finish_wait(&memcg_oom_waitq, &owait.wait);
  1857. mem_cgroup_out_of_memory(memcg, mask, order);
  1858. } else {
  1859. schedule();
  1860. finish_wait(&memcg_oom_waitq, &owait.wait);
  1861. }
  1862. spin_lock(&memcg_oom_lock);
  1863. if (locked)
  1864. mem_cgroup_oom_unlock(memcg);
  1865. memcg_wakeup_oom(memcg);
  1866. spin_unlock(&memcg_oom_lock);
  1867. mem_cgroup_unmark_under_oom(memcg);
  1868. if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current))
  1869. return false;
  1870. /* Give chance to dying process */
  1871. schedule_timeout_uninterruptible(1);
  1872. return true;
  1873. }
  1874. /*
  1875. * Currently used to update mapped file statistics, but the routine can be
  1876. * generalized to update other statistics as well.
  1877. *
  1878. * Notes: Race condition
  1879. *
  1880. * We usually use page_cgroup_lock() for accessing page_cgroup member but
  1881. * it tends to be costly. But considering some conditions, we doesn't need
  1882. * to do so _always_.
  1883. *
  1884. * Considering "charge", lock_page_cgroup() is not required because all
  1885. * file-stat operations happen after a page is attached to radix-tree. There
  1886. * are no race with "charge".
  1887. *
  1888. * Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup
  1889. * at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even
  1890. * if there are race with "uncharge". Statistics itself is properly handled
  1891. * by flags.
  1892. *
  1893. * Considering "move", this is an only case we see a race. To make the race
  1894. * small, we check mm->moving_account and detect there are possibility of race
  1895. * If there is, we take a lock.
  1896. */
  1897. void __mem_cgroup_begin_update_page_stat(struct page *page,
  1898. bool *locked, unsigned long *flags)
  1899. {
  1900. struct mem_cgroup *memcg;
  1901. struct page_cgroup *pc;
  1902. pc = lookup_page_cgroup(page);
  1903. again:
  1904. memcg = pc->mem_cgroup;
  1905. if (unlikely(!memcg || !PageCgroupUsed(pc)))
  1906. return;
  1907. /*
  1908. * If this memory cgroup is not under account moving, we don't
  1909. * need to take move_lock_mem_cgroup(). Because we already hold
  1910. * rcu_read_lock(), any calls to move_account will be delayed until
  1911. * rcu_read_unlock() if mem_cgroup_stolen() == true.
  1912. */
  1913. if (!mem_cgroup_stolen(memcg))
  1914. return;
  1915. move_lock_mem_cgroup(memcg, flags);
  1916. if (memcg != pc->mem_cgroup || !PageCgroupUsed(pc)) {
  1917. move_unlock_mem_cgroup(memcg, flags);
  1918. goto again;
  1919. }
  1920. *locked = true;
  1921. }
  1922. void __mem_cgroup_end_update_page_stat(struct page *page, unsigned long *flags)
  1923. {
  1924. struct page_cgroup *pc = lookup_page_cgroup(page);
  1925. /*
  1926. * It's guaranteed that pc->mem_cgroup never changes while
  1927. * lock is held because a routine modifies pc->mem_cgroup
  1928. * should take move_lock_mem_cgroup().
  1929. */
  1930. move_unlock_mem_cgroup(pc->mem_cgroup, flags);
  1931. }
  1932. void mem_cgroup_update_page_stat(struct page *page,
  1933. enum mem_cgroup_page_stat_item idx, int val)
  1934. {
  1935. struct mem_cgroup *memcg;
  1936. struct page_cgroup *pc = lookup_page_cgroup(page);
  1937. unsigned long uninitialized_var(flags);
  1938. if (mem_cgroup_disabled())
  1939. return;
  1940. memcg = pc->mem_cgroup;
  1941. if (unlikely(!memcg || !PageCgroupUsed(pc)))
  1942. return;
  1943. switch (idx) {
  1944. case MEMCG_NR_FILE_MAPPED:
  1945. idx = MEM_CGROUP_STAT_FILE_MAPPED;
  1946. break;
  1947. default:
  1948. BUG();
  1949. }
  1950. this_cpu_add(memcg->stat->count[idx], val);
  1951. }
  1952. /*
  1953. * size of first charge trial. "32" comes from vmscan.c's magic value.
  1954. * TODO: maybe necessary to use big numbers in big irons.
  1955. */
  1956. #define CHARGE_BATCH 32U
  1957. struct memcg_stock_pcp {
  1958. struct mem_cgroup *cached; /* this never be root cgroup */
  1959. unsigned int nr_pages;
  1960. struct work_struct work;
  1961. unsigned long flags;
  1962. #define FLUSHING_CACHED_CHARGE 0
  1963. };
  1964. static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
  1965. static DEFINE_MUTEX(percpu_charge_mutex);
  1966. /**
  1967. * consume_stock: Try to consume stocked charge on this cpu.
  1968. * @memcg: memcg to consume from.
  1969. * @nr_pages: how many pages to charge.
  1970. *
  1971. * The charges will only happen if @memcg matches the current cpu's memcg
  1972. * stock, and at least @nr_pages are available in that stock. Failure to
  1973. * service an allocation will refill the stock.
  1974. *
  1975. * returns true if successful, false otherwise.
  1976. */
  1977. static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
  1978. {
  1979. struct memcg_stock_pcp *stock;
  1980. bool ret = true;
  1981. if (nr_pages > CHARGE_BATCH)
  1982. return false;
  1983. stock = &get_cpu_var(memcg_stock);
  1984. if (memcg == stock->cached && stock->nr_pages >= nr_pages)
  1985. stock->nr_pages -= nr_pages;
  1986. else /* need to call res_counter_charge */
  1987. ret = false;
  1988. put_cpu_var(memcg_stock);
  1989. return ret;
  1990. }
  1991. /*
  1992. * Returns stocks cached in percpu to res_counter and reset cached information.
  1993. */
  1994. static void drain_stock(struct memcg_stock_pcp *stock)
  1995. {
  1996. struct mem_cgroup *old = stock->cached;
  1997. if (stock->nr_pages) {
  1998. unsigned long bytes = stock->nr_pages * PAGE_SIZE;
  1999. res_counter_uncharge(&old->res, bytes);
  2000. if (do_swap_account)
  2001. res_counter_uncharge(&old->memsw, bytes);
  2002. stock->nr_pages = 0;
  2003. }
  2004. stock->cached = NULL;
  2005. }
  2006. /*
  2007. * This must be called under preempt disabled or must be called by
  2008. * a thread which is pinned to local cpu.
  2009. */
  2010. static void drain_local_stock(struct work_struct *dummy)
  2011. {
  2012. struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
  2013. drain_stock(stock);
  2014. clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
  2015. }
  2016. static void __init memcg_stock_init(void)
  2017. {
  2018. int cpu;
  2019. for_each_possible_cpu(cpu) {
  2020. struct memcg_stock_pcp *stock =
  2021. &per_cpu(memcg_stock, cpu);
  2022. INIT_WORK(&stock->work, drain_local_stock);
  2023. }
  2024. }
  2025. /*
  2026. * Cache charges(val) which is from res_counter, to local per_cpu area.
  2027. * This will be consumed by consume_stock() function, later.
  2028. */
  2029. static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
  2030. {
  2031. struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
  2032. if (stock->cached != memcg) { /* reset if necessary */
  2033. drain_stock(stock);
  2034. stock->cached = memcg;
  2035. }
  2036. stock->nr_pages += nr_pages;
  2037. put_cpu_var(memcg_stock);
  2038. }
  2039. /*
  2040. * Drains all per-CPU charge caches for given root_memcg resp. subtree
  2041. * of the hierarchy under it. sync flag says whether we should block
  2042. * until the work is done.
  2043. */
  2044. static void drain_all_stock(struct mem_cgroup *root_memcg, bool sync)
  2045. {
  2046. int cpu, curcpu;
  2047. /* Notify other cpus that system-wide "drain" is running */
  2048. get_online_cpus();
  2049. curcpu = get_cpu();
  2050. for_each_online_cpu(cpu) {
  2051. struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
  2052. struct mem_cgroup *memcg;
  2053. memcg = stock->cached;
  2054. if (!memcg || !stock->nr_pages)
  2055. continue;
  2056. if (!mem_cgroup_same_or_subtree(root_memcg, memcg))
  2057. continue;
  2058. if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
  2059. if (cpu == curcpu)
  2060. drain_local_stock(&stock->work);
  2061. else
  2062. schedule_work_on(cpu, &stock->work);
  2063. }
  2064. }
  2065. put_cpu();
  2066. if (!sync)
  2067. goto out;
  2068. for_each_online_cpu(cpu) {
  2069. struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
  2070. if (test_bit(FLUSHING_CACHED_CHARGE, &stock->flags))
  2071. flush_work(&stock->work);
  2072. }
  2073. out:
  2074. put_online_cpus();
  2075. }
  2076. /*
  2077. * Tries to drain stocked charges in other cpus. This function is asynchronous
  2078. * and just put a work per cpu for draining localy on each cpu. Caller can
  2079. * expects some charges will be back to res_counter later but cannot wait for
  2080. * it.
  2081. */
  2082. static void drain_all_stock_async(struct mem_cgroup *root_memcg)
  2083. {
  2084. /*
  2085. * If someone calls draining, avoid adding more kworker runs.
  2086. */
  2087. if (!mutex_trylock(&percpu_charge_mutex))
  2088. return;
  2089. drain_all_stock(root_memcg, false);
  2090. mutex_unlock(&percpu_charge_mutex);
  2091. }
  2092. /* This is a synchronous drain interface. */
  2093. static void drain_all_stock_sync(struct mem_cgroup *root_memcg)
  2094. {
  2095. /* called when force_empty is called */
  2096. mutex_lock(&percpu_charge_mutex);
  2097. drain_all_stock(root_memcg, true);
  2098. mutex_unlock(&percpu_charge_mutex);
  2099. }
  2100. /*
  2101. * This function drains percpu counter value from DEAD cpu and
  2102. * move it to local cpu. Note that this function can be preempted.
  2103. */
  2104. static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *memcg, int cpu)
  2105. {
  2106. int i;
  2107. spin_lock(&memcg->pcp_counter_lock);
  2108. for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
  2109. long x = per_cpu(memcg->stat->count[i], cpu);
  2110. per_cpu(memcg->stat->count[i], cpu) = 0;
  2111. memcg->nocpu_base.count[i] += x;
  2112. }
  2113. for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
  2114. unsigned long x = per_cpu(memcg->stat->events[i], cpu);
  2115. per_cpu(memcg->stat->events[i], cpu) = 0;
  2116. memcg->nocpu_base.events[i] += x;
  2117. }
  2118. spin_unlock(&memcg->pcp_counter_lock);
  2119. }
  2120. static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb,
  2121. unsigned long action,
  2122. void *hcpu)
  2123. {
  2124. int cpu = (unsigned long)hcpu;
  2125. struct memcg_stock_pcp *stock;
  2126. struct mem_cgroup *iter;
  2127. if (action == CPU_ONLINE)
  2128. return NOTIFY_OK;
  2129. if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
  2130. return NOTIFY_OK;
  2131. for_each_mem_cgroup(iter)
  2132. mem_cgroup_drain_pcp_counter(iter, cpu);
  2133. stock = &per_cpu(memcg_stock, cpu);
  2134. drain_stock(stock);
  2135. return NOTIFY_OK;
  2136. }
  2137. /* See __mem_cgroup_try_charge() for details */
  2138. enum {
  2139. CHARGE_OK, /* success */
  2140. CHARGE_RETRY, /* need to retry but retry is not bad */
  2141. CHARGE_NOMEM, /* we can't do more. return -ENOMEM */
  2142. CHARGE_WOULDBLOCK, /* GFP_WAIT wasn't set and no enough res. */
  2143. CHARGE_OOM_DIE, /* the current is killed because of OOM */
  2144. };
  2145. static int mem_cgroup_do_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
  2146. unsigned int nr_pages, unsigned int min_pages,
  2147. bool oom_check)
  2148. {
  2149. unsigned long csize = nr_pages * PAGE_SIZE;
  2150. struct mem_cgroup *mem_over_limit;
  2151. struct res_counter *fail_res;
  2152. unsigned long flags = 0;
  2153. int ret;
  2154. ret = res_counter_charge(&memcg->res, csize, &fail_res);
  2155. if (likely(!ret)) {
  2156. if (!do_swap_account)
  2157. return CHARGE_OK;
  2158. ret = res_counter_charge(&memcg->memsw, csize, &fail_res);
  2159. if (likely(!ret))
  2160. return CHARGE_OK;
  2161. res_counter_uncharge(&memcg->res, csize);
  2162. mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw);
  2163. flags |= MEM_CGROUP_RECLAIM_NOSWAP;
  2164. } else
  2165. mem_over_limit = mem_cgroup_from_res_counter(fail_res, res);
  2166. /*
  2167. * Never reclaim on behalf of optional batching, retry with a
  2168. * single page instead.
  2169. */
  2170. if (nr_pages > min_pages)
  2171. return CHARGE_RETRY;
  2172. if (!(gfp_mask & __GFP_WAIT))
  2173. return CHARGE_WOULDBLOCK;
  2174. if (gfp_mask & __GFP_NORETRY)
  2175. return CHARGE_NOMEM;
  2176. ret = mem_cgroup_reclaim(mem_over_limit, gfp_mask, flags);
  2177. if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
  2178. return CHARGE_RETRY;
  2179. /*
  2180. * Even though the limit is exceeded at this point, reclaim
  2181. * may have been able to free some pages. Retry the charge
  2182. * before killing the task.
  2183. *
  2184. * Only for regular pages, though: huge pages are rather
  2185. * unlikely to succeed so close to the limit, and we fall back
  2186. * to regular pages anyway in case of failure.
  2187. */
  2188. if (nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER) && ret)
  2189. return CHARGE_RETRY;
  2190. /*
  2191. * At task move, charge accounts can be doubly counted. So, it's
  2192. * better to wait until the end of task_move if something is going on.
  2193. */
  2194. if (mem_cgroup_wait_acct_move(mem_over_limit))
  2195. return CHARGE_RETRY;
  2196. /* If we don't need to call oom-killer at el, return immediately */
  2197. if (!oom_check)
  2198. return CHARGE_NOMEM;
  2199. /* check OOM */
  2200. if (!mem_cgroup_handle_oom(mem_over_limit, gfp_mask, get_order(csize)))
  2201. return CHARGE_OOM_DIE;
  2202. return CHARGE_RETRY;
  2203. }
  2204. /*
  2205. * __mem_cgroup_try_charge() does
  2206. * 1. detect memcg to be charged against from passed *mm and *ptr,
  2207. * 2. update res_counter
  2208. * 3. call memory reclaim if necessary.
  2209. *
  2210. * In some special case, if the task is fatal, fatal_signal_pending() or
  2211. * has TIF_MEMDIE, this function returns -EINTR while writing root_mem_cgroup
  2212. * to *ptr. There are two reasons for this. 1: fatal threads should quit as soon
  2213. * as possible without any hazards. 2: all pages should have a valid
  2214. * pc->mem_cgroup. If mm is NULL and the caller doesn't pass a valid memcg
  2215. * pointer, that is treated as a charge to root_mem_cgroup.
  2216. *
  2217. * So __mem_cgroup_try_charge() will return
  2218. * 0 ... on success, filling *ptr with a valid memcg pointer.
  2219. * -ENOMEM ... charge failure because of resource limits.
  2220. * -EINTR ... if thread is fatal. *ptr is filled with root_mem_cgroup.
  2221. *
  2222. * Unlike the exported interface, an "oom" parameter is added. if oom==true,
  2223. * the oom-killer can be invoked.
  2224. */
  2225. static int __mem_cgroup_try_charge(struct mm_struct *mm,
  2226. gfp_t gfp_mask,
  2227. unsigned int nr_pages,
  2228. struct mem_cgroup **ptr,
  2229. bool oom)
  2230. {
  2231. unsigned int batch = max(CHARGE_BATCH, nr_pages);
  2232. int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
  2233. struct mem_cgroup *memcg = NULL;
  2234. int ret;
  2235. /*
  2236. * Unlike gloval-vm's OOM-kill, we're not in memory shortage
  2237. * in system level. So, allow to go ahead dying process in addition to
  2238. * MEMDIE process.
  2239. */
  2240. if (unlikely(test_thread_flag(TIF_MEMDIE)
  2241. || fatal_signal_pending(current)))
  2242. goto bypass;
  2243. /*
  2244. * We always charge the cgroup the mm_struct belongs to.
  2245. * The mm_struct's mem_cgroup changes on task migration if the
  2246. * thread group leader migrates. It's possible that mm is not
  2247. * set, if so charge the root memcg (happens for pagecache usage).
  2248. */
  2249. if (!*ptr && !mm)
  2250. *ptr = root_mem_cgroup;
  2251. again:
  2252. if (*ptr) { /* css should be a valid one */
  2253. memcg = *ptr;
  2254. if (mem_cgroup_is_root(memcg))
  2255. goto done;
  2256. if (consume_stock(memcg, nr_pages))
  2257. goto done;
  2258. css_get(&memcg->css);
  2259. } else {
  2260. struct task_struct *p;
  2261. rcu_read_lock();
  2262. p = rcu_dereference(mm->owner);
  2263. /*
  2264. * Because we don't have task_lock(), "p" can exit.
  2265. * In that case, "memcg" can point to root or p can be NULL with
  2266. * race with swapoff. Then, we have small risk of mis-accouning.
  2267. * But such kind of mis-account by race always happens because
  2268. * we don't have cgroup_mutex(). It's overkill and we allo that
  2269. * small race, here.
  2270. * (*) swapoff at el will charge against mm-struct not against
  2271. * task-struct. So, mm->owner can be NULL.
  2272. */
  2273. memcg = mem_cgroup_from_task(p);
  2274. if (!memcg)
  2275. memcg = root_mem_cgroup;
  2276. if (mem_cgroup_is_root(memcg)) {
  2277. rcu_read_unlock();
  2278. goto done;
  2279. }
  2280. if (consume_stock(memcg, nr_pages)) {
  2281. /*
  2282. * It seems dagerous to access memcg without css_get().
  2283. * But considering how consume_stok works, it's not
  2284. * necessary. If consume_stock success, some charges
  2285. * from this memcg are cached on this cpu. So, we
  2286. * don't need to call css_get()/css_tryget() before
  2287. * calling consume_stock().
  2288. */
  2289. rcu_read_unlock();
  2290. goto done;
  2291. }
  2292. /* after here, we may be blocked. we need to get refcnt */
  2293. if (!css_tryget(&memcg->css)) {
  2294. rcu_read_unlock();
  2295. goto again;
  2296. }
  2297. rcu_read_unlock();
  2298. }
  2299. do {
  2300. bool oom_check;
  2301. /* If killed, bypass charge */
  2302. if (fatal_signal_pending(current)) {
  2303. css_put(&memcg->css);
  2304. goto bypass;
  2305. }
  2306. oom_check = false;
  2307. if (oom && !nr_oom_retries) {
  2308. oom_check = true;
  2309. nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
  2310. }
  2311. ret = mem_cgroup_do_charge(memcg, gfp_mask, batch, nr_pages,
  2312. oom_check);
  2313. switch (ret) {
  2314. case CHARGE_OK:
  2315. break;
  2316. case CHARGE_RETRY: /* not in OOM situation but retry */
  2317. batch = nr_pages;
  2318. css_put(&memcg->css);
  2319. memcg = NULL;
  2320. goto again;
  2321. case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */
  2322. css_put(&memcg->css);
  2323. goto nomem;
  2324. case CHARGE_NOMEM: /* OOM routine works */
  2325. if (!oom) {
  2326. css_put(&memcg->css);
  2327. goto nomem;
  2328. }
  2329. /* If oom, we never return -ENOMEM */
  2330. nr_oom_retries--;
  2331. break;
  2332. case CHARGE_OOM_DIE: /* Killed by OOM Killer */
  2333. css_put(&memcg->css);
  2334. goto bypass;
  2335. }
  2336. } while (ret != CHARGE_OK);
  2337. if (batch > nr_pages)
  2338. refill_stock(memcg, batch - nr_pages);
  2339. css_put(&memcg->css);
  2340. done:
  2341. *ptr = memcg;
  2342. return 0;
  2343. nomem:
  2344. *ptr = NULL;
  2345. return -ENOMEM;
  2346. bypass:
  2347. *ptr = root_mem_cgroup;
  2348. return -EINTR;
  2349. }
  2350. /*
  2351. * Somemtimes we have to undo a charge we got by try_charge().
  2352. * This function is for that and do uncharge, put css's refcnt.
  2353. * gotten by try_charge().
  2354. */
  2355. static void __mem_cgroup_cancel_charge(struct mem_cgroup *memcg,
  2356. unsigned int nr_pages)
  2357. {
  2358. if (!mem_cgroup_is_root(memcg)) {
  2359. unsigned long bytes = nr_pages * PAGE_SIZE;
  2360. res_counter_uncharge(&memcg->res, bytes);
  2361. if (do_swap_account)
  2362. res_counter_uncharge(&memcg->memsw, bytes);
  2363. }
  2364. }
  2365. /*
  2366. * Cancel chrages in this cgroup....doesn't propagate to parent cgroup.
  2367. * This is useful when moving usage to parent cgroup.
  2368. */
  2369. static void __mem_cgroup_cancel_local_charge(struct mem_cgroup *memcg,
  2370. unsigned int nr_pages)
  2371. {
  2372. unsigned long bytes = nr_pages * PAGE_SIZE;
  2373. if (mem_cgroup_is_root(memcg))
  2374. return;
  2375. res_counter_uncharge_until(&memcg->res, memcg->res.parent, bytes);
  2376. if (do_swap_account)
  2377. res_counter_uncharge_until(&memcg->memsw,
  2378. memcg->memsw.parent, bytes);
  2379. }
  2380. /*
  2381. * A helper function to get mem_cgroup from ID. must be called under
  2382. * rcu_read_lock(). The caller is responsible for calling css_tryget if
  2383. * the mem_cgroup is used for charging. (dropping refcnt from swap can be
  2384. * called against removed memcg.)
  2385. */
  2386. static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
  2387. {
  2388. struct cgroup_subsys_state *css;
  2389. /* ID 0 is unused ID */
  2390. if (!id)
  2391. return NULL;
  2392. css = css_lookup(&mem_cgroup_subsys, id);
  2393. if (!css)
  2394. return NULL;
  2395. return mem_cgroup_from_css(css);
  2396. }
  2397. struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
  2398. {
  2399. struct mem_cgroup *memcg = NULL;
  2400. struct page_cgroup *pc;
  2401. unsigned short id;
  2402. swp_entry_t ent;
  2403. VM_BUG_ON(!PageLocked(page));
  2404. pc = lookup_page_cgroup(page);
  2405. lock_page_cgroup(pc);
  2406. if (PageCgroupUsed(pc)) {
  2407. memcg = pc->mem_cgroup;
  2408. if (memcg && !css_tryget(&memcg->css))
  2409. memcg = NULL;
  2410. } else if (PageSwapCache(page)) {
  2411. ent.val = page_private(page);
  2412. id = lookup_swap_cgroup_id(ent);
  2413. rcu_read_lock();
  2414. memcg = mem_cgroup_lookup(id);
  2415. if (memcg && !css_tryget(&memcg->css))
  2416. memcg = NULL;
  2417. rcu_read_unlock();
  2418. }
  2419. unlock_page_cgroup(pc);
  2420. return memcg;
  2421. }
  2422. static void __mem_cgroup_commit_charge(struct mem_cgroup *memcg,
  2423. struct page *page,
  2424. unsigned int nr_pages,
  2425. enum charge_type ctype,
  2426. bool lrucare)
  2427. {
  2428. struct page_cgroup *pc = lookup_page_cgroup(page);
  2429. struct zone *uninitialized_var(zone);
  2430. struct lruvec *lruvec;
  2431. bool was_on_lru = false;
  2432. bool anon;
  2433. lock_page_cgroup(pc);
  2434. VM_BUG_ON(PageCgroupUsed(pc));
  2435. /*
  2436. * we don't need page_cgroup_lock about tail pages, becase they are not
  2437. * accessed by any other context at this point.
  2438. */
  2439. /*
  2440. * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
  2441. * may already be on some other mem_cgroup's LRU. Take care of it.
  2442. */
  2443. if (lrucare) {
  2444. zone = page_zone(page);
  2445. spin_lock_irq(&zone->lru_lock);
  2446. if (PageLRU(page)) {
  2447. lruvec = mem_cgroup_zone_lruvec(zone, pc->mem_cgroup);
  2448. ClearPageLRU(page);
  2449. del_page_from_lru_list(page, lruvec, page_lru(page));
  2450. was_on_lru = true;
  2451. }
  2452. }
  2453. pc->mem_cgroup = memcg;
  2454. /*
  2455. * We access a page_cgroup asynchronously without lock_page_cgroup().
  2456. * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
  2457. * is accessed after testing USED bit. To make pc->mem_cgroup visible
  2458. * before USED bit, we need memory barrier here.
  2459. * See mem_cgroup_add_lru_list(), etc.
  2460. */
  2461. smp_wmb();
  2462. SetPageCgroupUsed(pc);
  2463. if (lrucare) {
  2464. if (was_on_lru) {
  2465. lruvec = mem_cgroup_zone_lruvec(zone, pc->mem_cgroup);
  2466. VM_BUG_ON(PageLRU(page));
  2467. SetPageLRU(page);
  2468. add_page_to_lru_list(page, lruvec, page_lru(page));
  2469. }
  2470. spin_unlock_irq(&zone->lru_lock);
  2471. }
  2472. if (ctype == MEM_CGROUP_CHARGE_TYPE_ANON)
  2473. anon = true;
  2474. else
  2475. anon = false;
  2476. mem_cgroup_charge_statistics(memcg, anon, nr_pages);
  2477. unlock_page_cgroup(pc);
  2478. /*
  2479. * "charge_statistics" updated event counter. Then, check it.
  2480. * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
  2481. * if they exceeds softlimit.
  2482. */
  2483. memcg_check_events(memcg, page);
  2484. }
  2485. static DEFINE_MUTEX(set_limit_mutex);
  2486. #ifdef CONFIG_MEMCG_KMEM
  2487. static inline bool memcg_can_account_kmem(struct mem_cgroup *memcg)
  2488. {
  2489. return !mem_cgroup_disabled() && !mem_cgroup_is_root(memcg) &&
  2490. (memcg->kmem_account_flags & KMEM_ACCOUNTED_MASK);
  2491. }
  2492. /*
  2493. * This is a bit cumbersome, but it is rarely used and avoids a backpointer
  2494. * in the memcg_cache_params struct.
  2495. */
  2496. static struct kmem_cache *memcg_params_to_cache(struct memcg_cache_params *p)
  2497. {
  2498. struct kmem_cache *cachep;
  2499. VM_BUG_ON(p->is_root_cache);
  2500. cachep = p->root_cache;
  2501. return cachep->memcg_params->memcg_caches[memcg_cache_id(p->memcg)];
  2502. }
  2503. #ifdef CONFIG_SLABINFO
  2504. static int mem_cgroup_slabinfo_read(struct cgroup *cont, struct cftype *cft,
  2505. struct seq_file *m)
  2506. {
  2507. struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
  2508. struct memcg_cache_params *params;
  2509. if (!memcg_can_account_kmem(memcg))
  2510. return -EIO;
  2511. print_slabinfo_header(m);
  2512. mutex_lock(&memcg->slab_caches_mutex);
  2513. list_for_each_entry(params, &memcg->memcg_slab_caches, list)
  2514. cache_show(memcg_params_to_cache(params), m);
  2515. mutex_unlock(&memcg->slab_caches_mutex);
  2516. return 0;
  2517. }
  2518. #endif
  2519. static int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp, u64 size)
  2520. {
  2521. struct res_counter *fail_res;
  2522. struct mem_cgroup *_memcg;
  2523. int ret = 0;
  2524. bool may_oom;
  2525. ret = res_counter_charge(&memcg->kmem, size, &fail_res);
  2526. if (ret)
  2527. return ret;
  2528. /*
  2529. * Conditions under which we can wait for the oom_killer. Those are
  2530. * the same conditions tested by the core page allocator
  2531. */
  2532. may_oom = (gfp & __GFP_FS) && !(gfp & __GFP_NORETRY);
  2533. _memcg = memcg;
  2534. ret = __mem_cgroup_try_charge(NULL, gfp, size >> PAGE_SHIFT,
  2535. &_memcg, may_oom);
  2536. if (ret == -EINTR) {
  2537. /*
  2538. * __mem_cgroup_try_charge() chosed to bypass to root due to
  2539. * OOM kill or fatal signal. Since our only options are to
  2540. * either fail the allocation or charge it to this cgroup, do
  2541. * it as a temporary condition. But we can't fail. From a
  2542. * kmem/slab perspective, the cache has already been selected,
  2543. * by mem_cgroup_kmem_get_cache(), so it is too late to change
  2544. * our minds.
  2545. *
  2546. * This condition will only trigger if the task entered
  2547. * memcg_charge_kmem in a sane state, but was OOM-killed during
  2548. * __mem_cgroup_try_charge() above. Tasks that were already
  2549. * dying when the allocation triggers should have been already
  2550. * directed to the root cgroup in memcontrol.h
  2551. */
  2552. res_counter_charge_nofail(&memcg->res, size, &fail_res);
  2553. if (do_swap_account)
  2554. res_counter_charge_nofail(&memcg->memsw, size,
  2555. &fail_res);
  2556. ret = 0;
  2557. } else if (ret)
  2558. res_counter_uncharge(&memcg->kmem, size);
  2559. return ret;
  2560. }
  2561. static void memcg_uncharge_kmem(struct mem_cgroup *memcg, u64 size)
  2562. {
  2563. res_counter_uncharge(&memcg->res, size);
  2564. if (do_swap_account)
  2565. res_counter_uncharge(&memcg->memsw, size);
  2566. /* Not down to 0 */
  2567. if (res_counter_uncharge(&memcg->kmem, size))
  2568. return;
  2569. if (memcg_kmem_test_and_clear_dead(memcg))
  2570. mem_cgroup_put(memcg);
  2571. }
  2572. void memcg_cache_list_add(struct mem_cgroup *memcg, struct kmem_cache *cachep)
  2573. {
  2574. if (!memcg)
  2575. return;
  2576. mutex_lock(&memcg->slab_caches_mutex);
  2577. list_add(&cachep->memcg_params->list, &memcg->memcg_slab_caches);
  2578. mutex_unlock(&memcg->slab_caches_mutex);
  2579. }
  2580. /*
  2581. * helper for acessing a memcg's index. It will be used as an index in the
  2582. * child cache array in kmem_cache, and also to derive its name. This function
  2583. * will return -1 when this is not a kmem-limited memcg.
  2584. */
  2585. int memcg_cache_id(struct mem_cgroup *memcg)
  2586. {
  2587. return memcg ? memcg->kmemcg_id : -1;
  2588. }
  2589. /*
  2590. * This ends up being protected by the set_limit mutex, during normal
  2591. * operation, because that is its main call site.
  2592. *
  2593. * But when we create a new cache, we can call this as well if its parent
  2594. * is kmem-limited. That will have to hold set_limit_mutex as well.
  2595. */
  2596. int memcg_update_cache_sizes(struct mem_cgroup *memcg)
  2597. {
  2598. int num, ret;
  2599. num = ida_simple_get(&kmem_limited_groups,
  2600. 0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
  2601. if (num < 0)
  2602. return num;
  2603. /*
  2604. * After this point, kmem_accounted (that we test atomically in
  2605. * the beginning of this conditional), is no longer 0. This
  2606. * guarantees only one process will set the following boolean
  2607. * to true. We don't need test_and_set because we're protected
  2608. * by the set_limit_mutex anyway.
  2609. */
  2610. memcg_kmem_set_activated(memcg);
  2611. ret = memcg_update_all_caches(num+1);
  2612. if (ret) {
  2613. ida_simple_remove(&kmem_limited_groups, num);
  2614. memcg_kmem_clear_activated(memcg);
  2615. return ret;
  2616. }
  2617. memcg->kmemcg_id = num;
  2618. INIT_LIST_HEAD(&memcg->memcg_slab_caches);
  2619. mutex_init(&memcg->slab_caches_mutex);
  2620. return 0;
  2621. }
  2622. static size_t memcg_caches_array_size(int num_groups)
  2623. {
  2624. ssize_t size;
  2625. if (num_groups <= 0)
  2626. return 0;
  2627. size = 2 * num_groups;
  2628. if (size < MEMCG_CACHES_MIN_SIZE)
  2629. size = MEMCG_CACHES_MIN_SIZE;
  2630. else if (size > MEMCG_CACHES_MAX_SIZE)
  2631. size = MEMCG_CACHES_MAX_SIZE;
  2632. return size;
  2633. }
  2634. /*
  2635. * We should update the current array size iff all caches updates succeed. This
  2636. * can only be done from the slab side. The slab mutex needs to be held when
  2637. * calling this.
  2638. */
  2639. void memcg_update_array_size(int num)
  2640. {
  2641. if (num > memcg_limited_groups_array_size)
  2642. memcg_limited_groups_array_size = memcg_caches_array_size(num);
  2643. }
  2644. static void kmem_cache_destroy_work_func(struct work_struct *w);
  2645. int memcg_update_cache_size(struct kmem_cache *s, int num_groups)
  2646. {
  2647. struct memcg_cache_params *cur_params = s->memcg_params;
  2648. VM_BUG_ON(s->memcg_params && !s->memcg_params->is_root_cache);
  2649. if (num_groups > memcg_limited_groups_array_size) {
  2650. int i;
  2651. ssize_t size = memcg_caches_array_size(num_groups);
  2652. size *= sizeof(void *);
  2653. size += sizeof(struct memcg_cache_params);
  2654. s->memcg_params = kzalloc(size, GFP_KERNEL);
  2655. if (!s->memcg_params) {
  2656. s->memcg_params = cur_params;
  2657. return -ENOMEM;
  2658. }
  2659. INIT_WORK(&s->memcg_params->destroy,
  2660. kmem_cache_destroy_work_func);
  2661. s->memcg_params->is_root_cache = true;
  2662. /*
  2663. * There is the chance it will be bigger than
  2664. * memcg_limited_groups_array_size, if we failed an allocation
  2665. * in a cache, in which case all caches updated before it, will
  2666. * have a bigger array.
  2667. *
  2668. * But if that is the case, the data after
  2669. * memcg_limited_groups_array_size is certainly unused
  2670. */
  2671. for (i = 0; i < memcg_limited_groups_array_size; i++) {
  2672. if (!cur_params->memcg_caches[i])
  2673. continue;
  2674. s->memcg_params->memcg_caches[i] =
  2675. cur_params->memcg_caches[i];
  2676. }
  2677. /*
  2678. * Ideally, we would wait until all caches succeed, and only
  2679. * then free the old one. But this is not worth the extra
  2680. * pointer per-cache we'd have to have for this.
  2681. *
  2682. * It is not a big deal if some caches are left with a size
  2683. * bigger than the others. And all updates will reset this
  2684. * anyway.
  2685. */
  2686. kfree(cur_params);
  2687. }
  2688. return 0;
  2689. }
  2690. int memcg_register_cache(struct mem_cgroup *memcg, struct kmem_cache *s,
  2691. struct kmem_cache *root_cache)
  2692. {
  2693. size_t size = sizeof(struct memcg_cache_params);
  2694. if (!memcg_kmem_enabled())
  2695. return 0;
  2696. if (!memcg)
  2697. size += memcg_limited_groups_array_size * sizeof(void *);
  2698. s->memcg_params = kzalloc(size, GFP_KERNEL);
  2699. if (!s->memcg_params)
  2700. return -ENOMEM;
  2701. INIT_WORK(&s->memcg_params->destroy,
  2702. kmem_cache_destroy_work_func);
  2703. if (memcg) {
  2704. s->memcg_params->memcg = memcg;
  2705. s->memcg_params->root_cache = root_cache;
  2706. } else
  2707. s->memcg_params->is_root_cache = true;
  2708. return 0;
  2709. }
  2710. void memcg_release_cache(struct kmem_cache *s)
  2711. {
  2712. struct kmem_cache *root;
  2713. struct mem_cgroup *memcg;
  2714. int id;
  2715. /*
  2716. * This happens, for instance, when a root cache goes away before we
  2717. * add any memcg.
  2718. */
  2719. if (!s->memcg_params)
  2720. return;
  2721. if (s->memcg_params->is_root_cache)
  2722. goto out;
  2723. memcg = s->memcg_params->memcg;
  2724. id = memcg_cache_id(memcg);
  2725. root = s->memcg_params->root_cache;
  2726. root->memcg_params->memcg_caches[id] = NULL;
  2727. mem_cgroup_put(memcg);
  2728. mutex_lock(&memcg->slab_caches_mutex);
  2729. list_del(&s->memcg_params->list);
  2730. mutex_unlock(&memcg->slab_caches_mutex);
  2731. out:
  2732. kfree(s->memcg_params);
  2733. }
  2734. /*
  2735. * During the creation a new cache, we need to disable our accounting mechanism
  2736. * altogether. This is true even if we are not creating, but rather just
  2737. * enqueing new caches to be created.
  2738. *
  2739. * This is because that process will trigger allocations; some visible, like
  2740. * explicit kmallocs to auxiliary data structures, name strings and internal
  2741. * cache structures; some well concealed, like INIT_WORK() that can allocate
  2742. * objects during debug.
  2743. *
  2744. * If any allocation happens during memcg_kmem_get_cache, we will recurse back
  2745. * to it. This may not be a bounded recursion: since the first cache creation
  2746. * failed to complete (waiting on the allocation), we'll just try to create the
  2747. * cache again, failing at the same point.
  2748. *
  2749. * memcg_kmem_get_cache is prepared to abort after seeing a positive count of
  2750. * memcg_kmem_skip_account. So we enclose anything that might allocate memory
  2751. * inside the following two functions.
  2752. */
  2753. static inline void memcg_stop_kmem_account(void)
  2754. {
  2755. VM_BUG_ON(!current->mm);
  2756. current->memcg_kmem_skip_account++;
  2757. }
  2758. static inline void memcg_resume_kmem_account(void)
  2759. {
  2760. VM_BUG_ON(!current->mm);
  2761. current->memcg_kmem_skip_account--;
  2762. }
  2763. static void kmem_cache_destroy_work_func(struct work_struct *w)
  2764. {
  2765. struct kmem_cache *cachep;
  2766. struct memcg_cache_params *p;
  2767. p = container_of(w, struct memcg_cache_params, destroy);
  2768. cachep = memcg_params_to_cache(p);
  2769. /*
  2770. * If we get down to 0 after shrink, we could delete right away.
  2771. * However, memcg_release_pages() already puts us back in the workqueue
  2772. * in that case. If we proceed deleting, we'll get a dangling
  2773. * reference, and removing the object from the workqueue in that case
  2774. * is unnecessary complication. We are not a fast path.
  2775. *
  2776. * Note that this case is fundamentally different from racing with
  2777. * shrink_slab(): if memcg_cgroup_destroy_cache() is called in
  2778. * kmem_cache_shrink, not only we would be reinserting a dead cache
  2779. * into the queue, but doing so from inside the worker racing to
  2780. * destroy it.
  2781. *
  2782. * So if we aren't down to zero, we'll just schedule a worker and try
  2783. * again
  2784. */
  2785. if (atomic_read(&cachep->memcg_params->nr_pages) != 0) {
  2786. kmem_cache_shrink(cachep);
  2787. if (atomic_read(&cachep->memcg_params->nr_pages) == 0)
  2788. return;
  2789. } else
  2790. kmem_cache_destroy(cachep);
  2791. }
  2792. void mem_cgroup_destroy_cache(struct kmem_cache *cachep)
  2793. {
  2794. if (!cachep->memcg_params->dead)
  2795. return;
  2796. /*
  2797. * There are many ways in which we can get here.
  2798. *
  2799. * We can get to a memory-pressure situation while the delayed work is
  2800. * still pending to run. The vmscan shrinkers can then release all
  2801. * cache memory and get us to destruction. If this is the case, we'll
  2802. * be executed twice, which is a bug (the second time will execute over
  2803. * bogus data). In this case, cancelling the work should be fine.
  2804. *
  2805. * But we can also get here from the worker itself, if
  2806. * kmem_cache_shrink is enough to shake all the remaining objects and
  2807. * get the page count to 0. In this case, we'll deadlock if we try to
  2808. * cancel the work (the worker runs with an internal lock held, which
  2809. * is the same lock we would hold for cancel_work_sync().)
  2810. *
  2811. * Since we can't possibly know who got us here, just refrain from
  2812. * running if there is already work pending
  2813. */
  2814. if (work_pending(&cachep->memcg_params->destroy))
  2815. return;
  2816. /*
  2817. * We have to defer the actual destroying to a workqueue, because
  2818. * we might currently be in a context that cannot sleep.
  2819. */
  2820. schedule_work(&cachep->memcg_params->destroy);
  2821. }
  2822. static char *memcg_cache_name(struct mem_cgroup *memcg, struct kmem_cache *s)
  2823. {
  2824. char *name;
  2825. struct dentry *dentry;
  2826. rcu_read_lock();
  2827. dentry = rcu_dereference(memcg->css.cgroup->dentry);
  2828. rcu_read_unlock();
  2829. BUG_ON(dentry == NULL);
  2830. name = kasprintf(GFP_KERNEL, "%s(%d:%s)", s->name,
  2831. memcg_cache_id(memcg), dentry->d_name.name);
  2832. return name;
  2833. }
  2834. static struct kmem_cache *kmem_cache_dup(struct mem_cgroup *memcg,
  2835. struct kmem_cache *s)
  2836. {
  2837. char *name;
  2838. struct kmem_cache *new;
  2839. name = memcg_cache_name(memcg, s);
  2840. if (!name)
  2841. return NULL;
  2842. new = kmem_cache_create_memcg(memcg, name, s->object_size, s->align,
  2843. (s->flags & ~SLAB_PANIC), s->ctor, s);
  2844. if (new)
  2845. new->allocflags |= __GFP_KMEMCG;
  2846. kfree(name);
  2847. return new;
  2848. }
  2849. /*
  2850. * This lock protects updaters, not readers. We want readers to be as fast as
  2851. * they can, and they will either see NULL or a valid cache value. Our model
  2852. * allow them to see NULL, in which case the root memcg will be selected.
  2853. *
  2854. * We need this lock because multiple allocations to the same cache from a non
  2855. * will span more than one worker. Only one of them can create the cache.
  2856. */
  2857. static DEFINE_MUTEX(memcg_cache_mutex);
  2858. static struct kmem_cache *memcg_create_kmem_cache(struct mem_cgroup *memcg,
  2859. struct kmem_cache *cachep)
  2860. {
  2861. struct kmem_cache *new_cachep;
  2862. int idx;
  2863. BUG_ON(!memcg_can_account_kmem(memcg));
  2864. idx = memcg_cache_id(memcg);
  2865. mutex_lock(&memcg_cache_mutex);
  2866. new_cachep = cachep->memcg_params->memcg_caches[idx];
  2867. if (new_cachep)
  2868. goto out;
  2869. new_cachep = kmem_cache_dup(memcg, cachep);
  2870. if (new_cachep == NULL) {
  2871. new_cachep = cachep;
  2872. goto out;
  2873. }
  2874. mem_cgroup_get(memcg);
  2875. atomic_set(&new_cachep->memcg_params->nr_pages , 0);
  2876. cachep->memcg_params->memcg_caches[idx] = new_cachep;
  2877. /*
  2878. * the readers won't lock, make sure everybody sees the updated value,
  2879. * so they won't put stuff in the queue again for no reason
  2880. */
  2881. wmb();
  2882. out:
  2883. mutex_unlock(&memcg_cache_mutex);
  2884. return new_cachep;
  2885. }
  2886. void kmem_cache_destroy_memcg_children(struct kmem_cache *s)
  2887. {
  2888. struct kmem_cache *c;
  2889. int i;
  2890. if (!s->memcg_params)
  2891. return;
  2892. if (!s->memcg_params->is_root_cache)
  2893. return;
  2894. /*
  2895. * If the cache is being destroyed, we trust that there is no one else
  2896. * requesting objects from it. Even if there are, the sanity checks in
  2897. * kmem_cache_destroy should caught this ill-case.
  2898. *
  2899. * Still, we don't want anyone else freeing memcg_caches under our
  2900. * noses, which can happen if a new memcg comes to life. As usual,
  2901. * we'll take the set_limit_mutex to protect ourselves against this.
  2902. */
  2903. mutex_lock(&set_limit_mutex);
  2904. for (i = 0; i < memcg_limited_groups_array_size; i++) {
  2905. c = s->memcg_params->memcg_caches[i];
  2906. if (!c)
  2907. continue;
  2908. /*
  2909. * We will now manually delete the caches, so to avoid races
  2910. * we need to cancel all pending destruction workers and
  2911. * proceed with destruction ourselves.
  2912. *
  2913. * kmem_cache_destroy() will call kmem_cache_shrink internally,
  2914. * and that could spawn the workers again: it is likely that
  2915. * the cache still have active pages until this very moment.
  2916. * This would lead us back to mem_cgroup_destroy_cache.
  2917. *
  2918. * But that will not execute at all if the "dead" flag is not
  2919. * set, so flip it down to guarantee we are in control.
  2920. */
  2921. c->memcg_params->dead = false;
  2922. cancel_work_sync(&c->memcg_params->destroy);
  2923. kmem_cache_destroy(c);
  2924. }
  2925. mutex_unlock(&set_limit_mutex);
  2926. }
  2927. struct create_work {
  2928. struct mem_cgroup *memcg;
  2929. struct kmem_cache *cachep;
  2930. struct work_struct work;
  2931. };
  2932. static void mem_cgroup_destroy_all_caches(struct mem_cgroup *memcg)
  2933. {
  2934. struct kmem_cache *cachep;
  2935. struct memcg_cache_params *params;
  2936. if (!memcg_kmem_is_active(memcg))
  2937. return;
  2938. mutex_lock(&memcg->slab_caches_mutex);
  2939. list_for_each_entry(params, &memcg->memcg_slab_caches, list) {
  2940. cachep = memcg_params_to_cache(params);
  2941. cachep->memcg_params->dead = true;
  2942. schedule_work(&cachep->memcg_params->destroy);
  2943. }
  2944. mutex_unlock(&memcg->slab_caches_mutex);
  2945. }
  2946. static void memcg_create_cache_work_func(struct work_struct *w)
  2947. {
  2948. struct create_work *cw;
  2949. cw = container_of(w, struct create_work, work);
  2950. memcg_create_kmem_cache(cw->memcg, cw->cachep);
  2951. /* Drop the reference gotten when we enqueued. */
  2952. css_put(&cw->memcg->css);
  2953. kfree(cw);
  2954. }
  2955. /*
  2956. * Enqueue the creation of a per-memcg kmem_cache.
  2957. * Called with rcu_read_lock.
  2958. */
  2959. static void __memcg_create_cache_enqueue(struct mem_cgroup *memcg,
  2960. struct kmem_cache *cachep)
  2961. {
  2962. struct create_work *cw;
  2963. cw = kmalloc(sizeof(struct create_work), GFP_NOWAIT);
  2964. if (cw == NULL)
  2965. return;
  2966. /* The corresponding put will be done in the workqueue. */
  2967. if (!css_tryget(&memcg->css)) {
  2968. kfree(cw);
  2969. return;
  2970. }
  2971. cw->memcg = memcg;
  2972. cw->cachep = cachep;
  2973. INIT_WORK(&cw->work, memcg_create_cache_work_func);
  2974. schedule_work(&cw->work);
  2975. }
  2976. static void memcg_create_cache_enqueue(struct mem_cgroup *memcg,
  2977. struct kmem_cache *cachep)
  2978. {
  2979. /*
  2980. * We need to stop accounting when we kmalloc, because if the
  2981. * corresponding kmalloc cache is not yet created, the first allocation
  2982. * in __memcg_create_cache_enqueue will recurse.
  2983. *
  2984. * However, it is better to enclose the whole function. Depending on
  2985. * the debugging options enabled, INIT_WORK(), for instance, can
  2986. * trigger an allocation. This too, will make us recurse. Because at
  2987. * this point we can't allow ourselves back into memcg_kmem_get_cache,
  2988. * the safest choice is to do it like this, wrapping the whole function.
  2989. */
  2990. memcg_stop_kmem_account();
  2991. __memcg_create_cache_enqueue(memcg, cachep);
  2992. memcg_resume_kmem_account();
  2993. }
  2994. /*
  2995. * Return the kmem_cache we're supposed to use for a slab allocation.
  2996. * We try to use the current memcg's version of the cache.
  2997. *
  2998. * If the cache does not exist yet, if we are the first user of it,
  2999. * we either create it immediately, if possible, or create it asynchronously
  3000. * in a workqueue.
  3001. * In the latter case, we will let the current allocation go through with
  3002. * the original cache.
  3003. *
  3004. * Can't be called in interrupt context or from kernel threads.
  3005. * This function needs to be called with rcu_read_lock() held.
  3006. */
  3007. struct kmem_cache *__memcg_kmem_get_cache(struct kmem_cache *cachep,
  3008. gfp_t gfp)
  3009. {
  3010. struct mem_cgroup *memcg;
  3011. int idx;
  3012. VM_BUG_ON(!cachep->memcg_params);
  3013. VM_BUG_ON(!cachep->memcg_params->is_root_cache);
  3014. if (!current->mm || current->memcg_kmem_skip_account)
  3015. return cachep;
  3016. rcu_read_lock();
  3017. memcg = mem_cgroup_from_task(rcu_dereference(current->mm->owner));
  3018. rcu_read_unlock();
  3019. if (!memcg_can_account_kmem(memcg))
  3020. return cachep;
  3021. idx = memcg_cache_id(memcg);
  3022. /*
  3023. * barrier to mare sure we're always seeing the up to date value. The
  3024. * code updating memcg_caches will issue a write barrier to match this.
  3025. */
  3026. read_barrier_depends();
  3027. if (unlikely(cachep->memcg_params->memcg_caches[idx] == NULL)) {
  3028. /*
  3029. * If we are in a safe context (can wait, and not in interrupt
  3030. * context), we could be be predictable and return right away.
  3031. * This would guarantee that the allocation being performed
  3032. * already belongs in the new cache.
  3033. *
  3034. * However, there are some clashes that can arrive from locking.
  3035. * For instance, because we acquire the slab_mutex while doing
  3036. * kmem_cache_dup, this means no further allocation could happen
  3037. * with the slab_mutex held.
  3038. *
  3039. * Also, because cache creation issue get_online_cpus(), this
  3040. * creates a lock chain: memcg_slab_mutex -> cpu_hotplug_mutex,
  3041. * that ends up reversed during cpu hotplug. (cpuset allocates
  3042. * a bunch of GFP_KERNEL memory during cpuup). Due to all that,
  3043. * better to defer everything.
  3044. */
  3045. memcg_create_cache_enqueue(memcg, cachep);
  3046. return cachep;
  3047. }
  3048. return cachep->memcg_params->memcg_caches[idx];
  3049. }
  3050. EXPORT_SYMBOL(__memcg_kmem_get_cache);
  3051. /*
  3052. * We need to verify if the allocation against current->mm->owner's memcg is
  3053. * possible for the given order. But the page is not allocated yet, so we'll
  3054. * need a further commit step to do the final arrangements.
  3055. *
  3056. * It is possible for the task to switch cgroups in this mean time, so at
  3057. * commit time, we can't rely on task conversion any longer. We'll then use
  3058. * the handle argument to return to the caller which cgroup we should commit
  3059. * against. We could also return the memcg directly and avoid the pointer
  3060. * passing, but a boolean return value gives better semantics considering
  3061. * the compiled-out case as well.
  3062. *
  3063. * Returning true means the allocation is possible.
  3064. */
  3065. bool
  3066. __memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **_memcg, int order)
  3067. {
  3068. struct mem_cgroup *memcg;
  3069. int ret;
  3070. *_memcg = NULL;
  3071. memcg = try_get_mem_cgroup_from_mm(current->mm);
  3072. /*
  3073. * very rare case described in mem_cgroup_from_task. Unfortunately there
  3074. * isn't much we can do without complicating this too much, and it would
  3075. * be gfp-dependent anyway. Just let it go
  3076. */
  3077. if (unlikely(!memcg))
  3078. return true;
  3079. if (!memcg_can_account_kmem(memcg)) {
  3080. css_put(&memcg->css);
  3081. return true;
  3082. }
  3083. ret = memcg_charge_kmem(memcg, gfp, PAGE_SIZE << order);
  3084. if (!ret)
  3085. *_memcg = memcg;
  3086. css_put(&memcg->css);
  3087. return (ret == 0);
  3088. }
  3089. void __memcg_kmem_commit_charge(struct page *page, struct mem_cgroup *memcg,
  3090. int order)
  3091. {
  3092. struct page_cgroup *pc;
  3093. VM_BUG_ON(mem_cgroup_is_root(memcg));
  3094. /* The page allocation failed. Revert */
  3095. if (!page) {
  3096. memcg_uncharge_kmem(memcg, PAGE_SIZE << order);
  3097. return;
  3098. }
  3099. pc = lookup_page_cgroup(page);
  3100. lock_page_cgroup(pc);
  3101. pc->mem_cgroup = memcg;
  3102. SetPageCgroupUsed(pc);
  3103. unlock_page_cgroup(pc);
  3104. }
  3105. void __memcg_kmem_uncharge_pages(struct page *page, int order)
  3106. {
  3107. struct mem_cgroup *memcg = NULL;
  3108. struct page_cgroup *pc;
  3109. pc = lookup_page_cgroup(page);
  3110. /*
  3111. * Fast unlocked return. Theoretically might have changed, have to
  3112. * check again after locking.
  3113. */
  3114. if (!PageCgroupUsed(pc))
  3115. return;
  3116. lock_page_cgroup(pc);
  3117. if (PageCgroupUsed(pc)) {
  3118. memcg = pc->mem_cgroup;
  3119. ClearPageCgroupUsed(pc);
  3120. }
  3121. unlock_page_cgroup(pc);
  3122. /*
  3123. * We trust that only if there is a memcg associated with the page, it
  3124. * is a valid allocation
  3125. */
  3126. if (!memcg)
  3127. return;
  3128. VM_BUG_ON(mem_cgroup_is_root(memcg));
  3129. memcg_uncharge_kmem(memcg, PAGE_SIZE << order);
  3130. }
  3131. #else
  3132. static inline void mem_cgroup_destroy_all_caches(struct mem_cgroup *memcg)
  3133. {
  3134. }
  3135. #endif /* CONFIG_MEMCG_KMEM */
  3136. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  3137. #define PCGF_NOCOPY_AT_SPLIT (1 << PCG_LOCK | 1 << PCG_MIGRATION)
  3138. /*
  3139. * Because tail pages are not marked as "used", set it. We're under
  3140. * zone->lru_lock, 'splitting on pmd' and compound_lock.
  3141. * charge/uncharge will be never happen and move_account() is done under
  3142. * compound_lock(), so we don't have to take care of races.
  3143. */
  3144. void mem_cgroup_split_huge_fixup(struct page *head)
  3145. {
  3146. struct page_cgroup *head_pc = lookup_page_cgroup(head);
  3147. struct page_cgroup *pc;
  3148. int i;
  3149. if (mem_cgroup_disabled())
  3150. return;
  3151. for (i = 1; i < HPAGE_PMD_NR; i++) {
  3152. pc = head_pc + i;
  3153. pc->mem_cgroup = head_pc->mem_cgroup;
  3154. smp_wmb();/* see __commit_charge() */
  3155. pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT;
  3156. }
  3157. }
  3158. #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  3159. /**
  3160. * mem_cgroup_move_account - move account of the page
  3161. * @page: the page
  3162. * @nr_pages: number of regular pages (>1 for huge pages)
  3163. * @pc: page_cgroup of the page.
  3164. * @from: mem_cgroup which the page is moved from.
  3165. * @to: mem_cgroup which the page is moved to. @from != @to.
  3166. *
  3167. * The caller must confirm following.
  3168. * - page is not on LRU (isolate_page() is useful.)
  3169. * - compound_lock is held when nr_pages > 1
  3170. *
  3171. * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
  3172. * from old cgroup.
  3173. */
  3174. static int mem_cgroup_move_account(struct page *page,
  3175. unsigned int nr_pages,
  3176. struct page_cgroup *pc,
  3177. struct mem_cgroup *from,
  3178. struct mem_cgroup *to)
  3179. {
  3180. unsigned long flags;
  3181. int ret;
  3182. bool anon = PageAnon(page);
  3183. VM_BUG_ON(from == to);
  3184. VM_BUG_ON(PageLRU(page));
  3185. /*
  3186. * The page is isolated from LRU. So, collapse function
  3187. * will not handle this page. But page splitting can happen.
  3188. * Do this check under compound_page_lock(). The caller should
  3189. * hold it.
  3190. */
  3191. ret = -EBUSY;
  3192. if (nr_pages > 1 && !PageTransHuge(page))
  3193. goto out;
  3194. lock_page_cgroup(pc);
  3195. ret = -EINVAL;
  3196. if (!PageCgroupUsed(pc) || pc->mem_cgroup != from)
  3197. goto unlock;
  3198. move_lock_mem_cgroup(from, &flags);
  3199. if (!anon && page_mapped(page)) {
  3200. /* Update mapped_file data for mem_cgroup */
  3201. preempt_disable();
  3202. __this_cpu_dec(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
  3203. __this_cpu_inc(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
  3204. preempt_enable();
  3205. }
  3206. mem_cgroup_charge_statistics(from, anon, -nr_pages);
  3207. /* caller should have done css_get */
  3208. pc->mem_cgroup = to;
  3209. mem_cgroup_charge_statistics(to, anon, nr_pages);
  3210. move_unlock_mem_cgroup(from, &flags);
  3211. ret = 0;
  3212. unlock:
  3213. unlock_page_cgroup(pc);
  3214. /*
  3215. * check events
  3216. */
  3217. memcg_check_events(to, page);
  3218. memcg_check_events(from, page);
  3219. out:
  3220. return ret;
  3221. }
  3222. /**
  3223. * mem_cgroup_move_parent - moves page to the parent group
  3224. * @page: the page to move
  3225. * @pc: page_cgroup of the page
  3226. * @child: page's cgroup
  3227. *
  3228. * move charges to its parent or the root cgroup if the group has no
  3229. * parent (aka use_hierarchy==0).
  3230. * Although this might fail (get_page_unless_zero, isolate_lru_page or
  3231. * mem_cgroup_move_account fails) the failure is always temporary and
  3232. * it signals a race with a page removal/uncharge or migration. In the
  3233. * first case the page is on the way out and it will vanish from the LRU
  3234. * on the next attempt and the call should be retried later.
  3235. * Isolation from the LRU fails only if page has been isolated from
  3236. * the LRU since we looked at it and that usually means either global
  3237. * reclaim or migration going on. The page will either get back to the
  3238. * LRU or vanish.
  3239. * Finaly mem_cgroup_move_account fails only if the page got uncharged
  3240. * (!PageCgroupUsed) or moved to a different group. The page will
  3241. * disappear in the next attempt.
  3242. */
  3243. static int mem_cgroup_move_parent(struct page *page,
  3244. struct page_cgroup *pc,
  3245. struct mem_cgroup *child)
  3246. {
  3247. struct mem_cgroup *parent;
  3248. unsigned int nr_pages;
  3249. unsigned long uninitialized_var(flags);
  3250. int ret;
  3251. VM_BUG_ON(mem_cgroup_is_root(child));
  3252. ret = -EBUSY;
  3253. if (!get_page_unless_zero(page))
  3254. goto out;
  3255. if (isolate_lru_page(page))
  3256. goto put;
  3257. nr_pages = hpage_nr_pages(page);
  3258. parent = parent_mem_cgroup(child);
  3259. /*
  3260. * If no parent, move charges to root cgroup.
  3261. */
  3262. if (!parent)
  3263. parent = root_mem_cgroup;
  3264. if (nr_pages > 1) {
  3265. VM_BUG_ON(!PageTransHuge(page));
  3266. flags = compound_lock_irqsave(page);
  3267. }
  3268. ret = mem_cgroup_move_account(page, nr_pages,
  3269. pc, child, parent);
  3270. if (!ret)
  3271. __mem_cgroup_cancel_local_charge(child, nr_pages);
  3272. if (nr_pages > 1)
  3273. compound_unlock_irqrestore(page, flags);
  3274. putback_lru_page(page);
  3275. put:
  3276. put_page(page);
  3277. out:
  3278. return ret;
  3279. }
  3280. /*
  3281. * Charge the memory controller for page usage.
  3282. * Return
  3283. * 0 if the charge was successful
  3284. * < 0 if the cgroup is over its limit
  3285. */
  3286. static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
  3287. gfp_t gfp_mask, enum charge_type ctype)
  3288. {
  3289. struct mem_cgroup *memcg = NULL;
  3290. unsigned int nr_pages = 1;
  3291. bool oom = true;
  3292. int ret;
  3293. if (PageTransHuge(page)) {
  3294. nr_pages <<= compound_order(page);
  3295. VM_BUG_ON(!PageTransHuge(page));
  3296. /*
  3297. * Never OOM-kill a process for a huge page. The
  3298. * fault handler will fall back to regular pages.
  3299. */
  3300. oom = false;
  3301. }
  3302. ret = __mem_cgroup_try_charge(mm, gfp_mask, nr_pages, &memcg, oom);
  3303. if (ret == -ENOMEM)
  3304. return ret;
  3305. __mem_cgroup_commit_charge(memcg, page, nr_pages, ctype, false);
  3306. return 0;
  3307. }
  3308. int mem_cgroup_newpage_charge(struct page *page,
  3309. struct mm_struct *mm, gfp_t gfp_mask)
  3310. {
  3311. if (mem_cgroup_disabled())
  3312. return 0;
  3313. VM_BUG_ON(page_mapped(page));
  3314. VM_BUG_ON(page->mapping && !PageAnon(page));
  3315. VM_BUG_ON(!mm);
  3316. return mem_cgroup_charge_common(page, mm, gfp_mask,
  3317. MEM_CGROUP_CHARGE_TYPE_ANON);
  3318. }
  3319. /*
  3320. * While swap-in, try_charge -> commit or cancel, the page is locked.
  3321. * And when try_charge() successfully returns, one refcnt to memcg without
  3322. * struct page_cgroup is acquired. This refcnt will be consumed by
  3323. * "commit()" or removed by "cancel()"
  3324. */
  3325. static int __mem_cgroup_try_charge_swapin(struct mm_struct *mm,
  3326. struct page *page,
  3327. gfp_t mask,
  3328. struct mem_cgroup **memcgp)
  3329. {
  3330. struct mem_cgroup *memcg;
  3331. struct page_cgroup *pc;
  3332. int ret;
  3333. pc = lookup_page_cgroup(page);
  3334. /*
  3335. * Every swap fault against a single page tries to charge the
  3336. * page, bail as early as possible. shmem_unuse() encounters
  3337. * already charged pages, too. The USED bit is protected by
  3338. * the page lock, which serializes swap cache removal, which
  3339. * in turn serializes uncharging.
  3340. */
  3341. if (PageCgroupUsed(pc))
  3342. return 0;
  3343. if (!do_swap_account)
  3344. goto charge_cur_mm;
  3345. memcg = try_get_mem_cgroup_from_page(page);
  3346. if (!memcg)
  3347. goto charge_cur_mm;
  3348. *memcgp = memcg;
  3349. ret = __mem_cgroup_try_charge(NULL, mask, 1, memcgp, true);
  3350. css_put(&memcg->css);
  3351. if (ret == -EINTR)
  3352. ret = 0;
  3353. return ret;
  3354. charge_cur_mm:
  3355. ret = __mem_cgroup_try_charge(mm, mask, 1, memcgp, true);
  3356. if (ret == -EINTR)
  3357. ret = 0;
  3358. return ret;
  3359. }
  3360. int mem_cgroup_try_charge_swapin(struct mm_struct *mm, struct page *page,
  3361. gfp_t gfp_mask, struct mem_cgroup **memcgp)
  3362. {
  3363. *memcgp = NULL;
  3364. if (mem_cgroup_disabled())
  3365. return 0;
  3366. /*
  3367. * A racing thread's fault, or swapoff, may have already
  3368. * updated the pte, and even removed page from swap cache: in
  3369. * those cases unuse_pte()'s pte_same() test will fail; but
  3370. * there's also a KSM case which does need to charge the page.
  3371. */
  3372. if (!PageSwapCache(page)) {
  3373. int ret;
  3374. ret = __mem_cgroup_try_charge(mm, gfp_mask, 1, memcgp, true);
  3375. if (ret == -EINTR)
  3376. ret = 0;
  3377. return ret;
  3378. }
  3379. return __mem_cgroup_try_charge_swapin(mm, page, gfp_mask, memcgp);
  3380. }
  3381. void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *memcg)
  3382. {
  3383. if (mem_cgroup_disabled())
  3384. return;
  3385. if (!memcg)
  3386. return;
  3387. __mem_cgroup_cancel_charge(memcg, 1);
  3388. }
  3389. static void
  3390. __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *memcg,
  3391. enum charge_type ctype)
  3392. {
  3393. if (mem_cgroup_disabled())
  3394. return;
  3395. if (!memcg)
  3396. return;
  3397. __mem_cgroup_commit_charge(memcg, page, 1, ctype, true);
  3398. /*
  3399. * Now swap is on-memory. This means this page may be
  3400. * counted both as mem and swap....double count.
  3401. * Fix it by uncharging from memsw. Basically, this SwapCache is stable
  3402. * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
  3403. * may call delete_from_swap_cache() before reach here.
  3404. */
  3405. if (do_swap_account && PageSwapCache(page)) {
  3406. swp_entry_t ent = {.val = page_private(page)};
  3407. mem_cgroup_uncharge_swap(ent);
  3408. }
  3409. }
  3410. void mem_cgroup_commit_charge_swapin(struct page *page,
  3411. struct mem_cgroup *memcg)
  3412. {
  3413. __mem_cgroup_commit_charge_swapin(page, memcg,
  3414. MEM_CGROUP_CHARGE_TYPE_ANON);
  3415. }
  3416. int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
  3417. gfp_t gfp_mask)
  3418. {
  3419. struct mem_cgroup *memcg = NULL;
  3420. enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
  3421. int ret;
  3422. if (mem_cgroup_disabled())
  3423. return 0;
  3424. if (PageCompound(page))
  3425. return 0;
  3426. if (!PageSwapCache(page))
  3427. ret = mem_cgroup_charge_common(page, mm, gfp_mask, type);
  3428. else { /* page is swapcache/shmem */
  3429. ret = __mem_cgroup_try_charge_swapin(mm, page,
  3430. gfp_mask, &memcg);
  3431. if (!ret)
  3432. __mem_cgroup_commit_charge_swapin(page, memcg, type);
  3433. }
  3434. return ret;
  3435. }
  3436. static void mem_cgroup_do_uncharge(struct mem_cgroup *memcg,
  3437. unsigned int nr_pages,
  3438. const enum charge_type ctype)
  3439. {
  3440. struct memcg_batch_info *batch = NULL;
  3441. bool uncharge_memsw = true;
  3442. /* If swapout, usage of swap doesn't decrease */
  3443. if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
  3444. uncharge_memsw = false;
  3445. batch = &current->memcg_batch;
  3446. /*
  3447. * In usual, we do css_get() when we remember memcg pointer.
  3448. * But in this case, we keep res->usage until end of a series of
  3449. * uncharges. Then, it's ok to ignore memcg's refcnt.
  3450. */
  3451. if (!batch->memcg)
  3452. batch->memcg = memcg;
  3453. /*
  3454. * do_batch > 0 when unmapping pages or inode invalidate/truncate.
  3455. * In those cases, all pages freed continuously can be expected to be in
  3456. * the same cgroup and we have chance to coalesce uncharges.
  3457. * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE)
  3458. * because we want to do uncharge as soon as possible.
  3459. */
  3460. if (!batch->do_batch || test_thread_flag(TIF_MEMDIE))
  3461. goto direct_uncharge;
  3462. if (nr_pages > 1)
  3463. goto direct_uncharge;
  3464. /*
  3465. * In typical case, batch->memcg == mem. This means we can
  3466. * merge a series of uncharges to an uncharge of res_counter.
  3467. * If not, we uncharge res_counter ony by one.
  3468. */
  3469. if (batch->memcg != memcg)
  3470. goto direct_uncharge;
  3471. /* remember freed charge and uncharge it later */
  3472. batch->nr_pages++;
  3473. if (uncharge_memsw)
  3474. batch->memsw_nr_pages++;
  3475. return;
  3476. direct_uncharge:
  3477. res_counter_uncharge(&memcg->res, nr_pages * PAGE_SIZE);
  3478. if (uncharge_memsw)
  3479. res_counter_uncharge(&memcg->memsw, nr_pages * PAGE_SIZE);
  3480. if (unlikely(batch->memcg != memcg))
  3481. memcg_oom_recover(memcg);
  3482. }
  3483. /*
  3484. * uncharge if !page_mapped(page)
  3485. */
  3486. static struct mem_cgroup *
  3487. __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype,
  3488. bool end_migration)
  3489. {
  3490. struct mem_cgroup *memcg = NULL;
  3491. unsigned int nr_pages = 1;
  3492. struct page_cgroup *pc;
  3493. bool anon;
  3494. if (mem_cgroup_disabled())
  3495. return NULL;
  3496. VM_BUG_ON(PageSwapCache(page));
  3497. if (PageTransHuge(page)) {
  3498. nr_pages <<= compound_order(page);
  3499. VM_BUG_ON(!PageTransHuge(page));
  3500. }
  3501. /*
  3502. * Check if our page_cgroup is valid
  3503. */
  3504. pc = lookup_page_cgroup(page);
  3505. if (unlikely(!PageCgroupUsed(pc)))
  3506. return NULL;
  3507. lock_page_cgroup(pc);
  3508. memcg = pc->mem_cgroup;
  3509. if (!PageCgroupUsed(pc))
  3510. goto unlock_out;
  3511. anon = PageAnon(page);
  3512. switch (ctype) {
  3513. case MEM_CGROUP_CHARGE_TYPE_ANON:
  3514. /*
  3515. * Generally PageAnon tells if it's the anon statistics to be
  3516. * updated; but sometimes e.g. mem_cgroup_uncharge_page() is
  3517. * used before page reached the stage of being marked PageAnon.
  3518. */
  3519. anon = true;
  3520. /* fallthrough */
  3521. case MEM_CGROUP_CHARGE_TYPE_DROP:
  3522. /* See mem_cgroup_prepare_migration() */
  3523. if (page_mapped(page))
  3524. goto unlock_out;
  3525. /*
  3526. * Pages under migration may not be uncharged. But
  3527. * end_migration() /must/ be the one uncharging the
  3528. * unused post-migration page and so it has to call
  3529. * here with the migration bit still set. See the
  3530. * res_counter handling below.
  3531. */
  3532. if (!end_migration && PageCgroupMigration(pc))
  3533. goto unlock_out;
  3534. break;
  3535. case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
  3536. if (!PageAnon(page)) { /* Shared memory */
  3537. if (page->mapping && !page_is_file_cache(page))
  3538. goto unlock_out;
  3539. } else if (page_mapped(page)) /* Anon */
  3540. goto unlock_out;
  3541. break;
  3542. default:
  3543. break;
  3544. }
  3545. mem_cgroup_charge_statistics(memcg, anon, -nr_pages);
  3546. ClearPageCgroupUsed(pc);
  3547. /*
  3548. * pc->mem_cgroup is not cleared here. It will be accessed when it's
  3549. * freed from LRU. This is safe because uncharged page is expected not
  3550. * to be reused (freed soon). Exception is SwapCache, it's handled by
  3551. * special functions.
  3552. */
  3553. unlock_page_cgroup(pc);
  3554. /*
  3555. * even after unlock, we have memcg->res.usage here and this memcg
  3556. * will never be freed.
  3557. */
  3558. memcg_check_events(memcg, page);
  3559. if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) {
  3560. mem_cgroup_swap_statistics(memcg, true);
  3561. mem_cgroup_get(memcg);
  3562. }
  3563. /*
  3564. * Migration does not charge the res_counter for the
  3565. * replacement page, so leave it alone when phasing out the
  3566. * page that is unused after the migration.
  3567. */
  3568. if (!end_migration && !mem_cgroup_is_root(memcg))
  3569. mem_cgroup_do_uncharge(memcg, nr_pages, ctype);
  3570. return memcg;
  3571. unlock_out:
  3572. unlock_page_cgroup(pc);
  3573. return NULL;
  3574. }
  3575. void mem_cgroup_uncharge_page(struct page *page)
  3576. {
  3577. /* early check. */
  3578. if (page_mapped(page))
  3579. return;
  3580. VM_BUG_ON(page->mapping && !PageAnon(page));
  3581. if (PageSwapCache(page))
  3582. return;
  3583. __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_ANON, false);
  3584. }
  3585. void mem_cgroup_uncharge_cache_page(struct page *page)
  3586. {
  3587. VM_BUG_ON(page_mapped(page));
  3588. VM_BUG_ON(page->mapping);
  3589. __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE, false);
  3590. }
  3591. /*
  3592. * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate.
  3593. * In that cases, pages are freed continuously and we can expect pages
  3594. * are in the same memcg. All these calls itself limits the number of
  3595. * pages freed at once, then uncharge_start/end() is called properly.
  3596. * This may be called prural(2) times in a context,
  3597. */
  3598. void mem_cgroup_uncharge_start(void)
  3599. {
  3600. current->memcg_batch.do_batch++;
  3601. /* We can do nest. */
  3602. if (current->memcg_batch.do_batch == 1) {
  3603. current->memcg_batch.memcg = NULL;
  3604. current->memcg_batch.nr_pages = 0;
  3605. current->memcg_batch.memsw_nr_pages = 0;
  3606. }
  3607. }
  3608. void mem_cgroup_uncharge_end(void)
  3609. {
  3610. struct memcg_batch_info *batch = &current->memcg_batch;
  3611. if (!batch->do_batch)
  3612. return;
  3613. batch->do_batch--;
  3614. if (batch->do_batch) /* If stacked, do nothing. */
  3615. return;
  3616. if (!batch->memcg)
  3617. return;
  3618. /*
  3619. * This "batch->memcg" is valid without any css_get/put etc...
  3620. * bacause we hide charges behind us.
  3621. */
  3622. if (batch->nr_pages)
  3623. res_counter_uncharge(&batch->memcg->res,
  3624. batch->nr_pages * PAGE_SIZE);
  3625. if (batch->memsw_nr_pages)
  3626. res_counter_uncharge(&batch->memcg->memsw,
  3627. batch->memsw_nr_pages * PAGE_SIZE);
  3628. memcg_oom_recover(batch->memcg);
  3629. /* forget this pointer (for sanity check) */
  3630. batch->memcg = NULL;
  3631. }
  3632. #ifdef CONFIG_SWAP
  3633. /*
  3634. * called after __delete_from_swap_cache() and drop "page" account.
  3635. * memcg information is recorded to swap_cgroup of "ent"
  3636. */
  3637. void
  3638. mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
  3639. {
  3640. struct mem_cgroup *memcg;
  3641. int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT;
  3642. if (!swapout) /* this was a swap cache but the swap is unused ! */
  3643. ctype = MEM_CGROUP_CHARGE_TYPE_DROP;
  3644. memcg = __mem_cgroup_uncharge_common(page, ctype, false);
  3645. /*
  3646. * record memcg information, if swapout && memcg != NULL,
  3647. * mem_cgroup_get() was called in uncharge().
  3648. */
  3649. if (do_swap_account && swapout && memcg)
  3650. swap_cgroup_record(ent, css_id(&memcg->css));
  3651. }
  3652. #endif
  3653. #ifdef CONFIG_MEMCG_SWAP
  3654. /*
  3655. * called from swap_entry_free(). remove record in swap_cgroup and
  3656. * uncharge "memsw" account.
  3657. */
  3658. void mem_cgroup_uncharge_swap(swp_entry_t ent)
  3659. {
  3660. struct mem_cgroup *memcg;
  3661. unsigned short id;
  3662. if (!do_swap_account)
  3663. return;
  3664. id = swap_cgroup_record(ent, 0);
  3665. rcu_read_lock();
  3666. memcg = mem_cgroup_lookup(id);
  3667. if (memcg) {
  3668. /*
  3669. * We uncharge this because swap is freed.
  3670. * This memcg can be obsolete one. We avoid calling css_tryget
  3671. */
  3672. if (!mem_cgroup_is_root(memcg))
  3673. res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
  3674. mem_cgroup_swap_statistics(memcg, false);
  3675. mem_cgroup_put(memcg);
  3676. }
  3677. rcu_read_unlock();
  3678. }
  3679. /**
  3680. * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
  3681. * @entry: swap entry to be moved
  3682. * @from: mem_cgroup which the entry is moved from
  3683. * @to: mem_cgroup which the entry is moved to
  3684. *
  3685. * It succeeds only when the swap_cgroup's record for this entry is the same
  3686. * as the mem_cgroup's id of @from.
  3687. *
  3688. * Returns 0 on success, -EINVAL on failure.
  3689. *
  3690. * The caller must have charged to @to, IOW, called res_counter_charge() about
  3691. * both res and memsw, and called css_get().
  3692. */
  3693. static int mem_cgroup_move_swap_account(swp_entry_t entry,
  3694. struct mem_cgroup *from, struct mem_cgroup *to)
  3695. {
  3696. unsigned short old_id, new_id;
  3697. old_id = css_id(&from->css);
  3698. new_id = css_id(&to->css);
  3699. if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
  3700. mem_cgroup_swap_statistics(from, false);
  3701. mem_cgroup_swap_statistics(to, true);
  3702. /*
  3703. * This function is only called from task migration context now.
  3704. * It postpones res_counter and refcount handling till the end
  3705. * of task migration(mem_cgroup_clear_mc()) for performance
  3706. * improvement. But we cannot postpone mem_cgroup_get(to)
  3707. * because if the process that has been moved to @to does
  3708. * swap-in, the refcount of @to might be decreased to 0.
  3709. */
  3710. mem_cgroup_get(to);
  3711. return 0;
  3712. }
  3713. return -EINVAL;
  3714. }
  3715. #else
  3716. static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
  3717. struct mem_cgroup *from, struct mem_cgroup *to)
  3718. {
  3719. return -EINVAL;
  3720. }
  3721. #endif
  3722. /*
  3723. * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
  3724. * page belongs to.
  3725. */
  3726. void mem_cgroup_prepare_migration(struct page *page, struct page *newpage,
  3727. struct mem_cgroup **memcgp)
  3728. {
  3729. struct mem_cgroup *memcg = NULL;
  3730. unsigned int nr_pages = 1;
  3731. struct page_cgroup *pc;
  3732. enum charge_type ctype;
  3733. *memcgp = NULL;
  3734. if (mem_cgroup_disabled())
  3735. return;
  3736. if (PageTransHuge(page))
  3737. nr_pages <<= compound_order(page);
  3738. pc = lookup_page_cgroup(page);
  3739. lock_page_cgroup(pc);
  3740. if (PageCgroupUsed(pc)) {
  3741. memcg = pc->mem_cgroup;
  3742. css_get(&memcg->css);
  3743. /*
  3744. * At migrating an anonymous page, its mapcount goes down
  3745. * to 0 and uncharge() will be called. But, even if it's fully
  3746. * unmapped, migration may fail and this page has to be
  3747. * charged again. We set MIGRATION flag here and delay uncharge
  3748. * until end_migration() is called
  3749. *
  3750. * Corner Case Thinking
  3751. * A)
  3752. * When the old page was mapped as Anon and it's unmap-and-freed
  3753. * while migration was ongoing.
  3754. * If unmap finds the old page, uncharge() of it will be delayed
  3755. * until end_migration(). If unmap finds a new page, it's
  3756. * uncharged when it make mapcount to be 1->0. If unmap code
  3757. * finds swap_migration_entry, the new page will not be mapped
  3758. * and end_migration() will find it(mapcount==0).
  3759. *
  3760. * B)
  3761. * When the old page was mapped but migraion fails, the kernel
  3762. * remaps it. A charge for it is kept by MIGRATION flag even
  3763. * if mapcount goes down to 0. We can do remap successfully
  3764. * without charging it again.
  3765. *
  3766. * C)
  3767. * The "old" page is under lock_page() until the end of
  3768. * migration, so, the old page itself will not be swapped-out.
  3769. * If the new page is swapped out before end_migraton, our
  3770. * hook to usual swap-out path will catch the event.
  3771. */
  3772. if (PageAnon(page))
  3773. SetPageCgroupMigration(pc);
  3774. }
  3775. unlock_page_cgroup(pc);
  3776. /*
  3777. * If the page is not charged at this point,
  3778. * we return here.
  3779. */
  3780. if (!memcg)
  3781. return;
  3782. *memcgp = memcg;
  3783. /*
  3784. * We charge new page before it's used/mapped. So, even if unlock_page()
  3785. * is called before end_migration, we can catch all events on this new
  3786. * page. In the case new page is migrated but not remapped, new page's
  3787. * mapcount will be finally 0 and we call uncharge in end_migration().
  3788. */
  3789. if (PageAnon(page))
  3790. ctype = MEM_CGROUP_CHARGE_TYPE_ANON;
  3791. else
  3792. ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
  3793. /*
  3794. * The page is committed to the memcg, but it's not actually
  3795. * charged to the res_counter since we plan on replacing the
  3796. * old one and only one page is going to be left afterwards.
  3797. */
  3798. __mem_cgroup_commit_charge(memcg, newpage, nr_pages, ctype, false);
  3799. }
  3800. /* remove redundant charge if migration failed*/
  3801. void mem_cgroup_end_migration(struct mem_cgroup *memcg,
  3802. struct page *oldpage, struct page *newpage, bool migration_ok)
  3803. {
  3804. struct page *used, *unused;
  3805. struct page_cgroup *pc;
  3806. bool anon;
  3807. if (!memcg)
  3808. return;
  3809. if (!migration_ok) {
  3810. used = oldpage;
  3811. unused = newpage;
  3812. } else {
  3813. used = newpage;
  3814. unused = oldpage;
  3815. }
  3816. anon = PageAnon(used);
  3817. __mem_cgroup_uncharge_common(unused,
  3818. anon ? MEM_CGROUP_CHARGE_TYPE_ANON
  3819. : MEM_CGROUP_CHARGE_TYPE_CACHE,
  3820. true);
  3821. css_put(&memcg->css);
  3822. /*
  3823. * We disallowed uncharge of pages under migration because mapcount
  3824. * of the page goes down to zero, temporarly.
  3825. * Clear the flag and check the page should be charged.
  3826. */
  3827. pc = lookup_page_cgroup(oldpage);
  3828. lock_page_cgroup(pc);
  3829. ClearPageCgroupMigration(pc);
  3830. unlock_page_cgroup(pc);
  3831. /*
  3832. * If a page is a file cache, radix-tree replacement is very atomic
  3833. * and we can skip this check. When it was an Anon page, its mapcount
  3834. * goes down to 0. But because we added MIGRATION flage, it's not
  3835. * uncharged yet. There are several case but page->mapcount check
  3836. * and USED bit check in mem_cgroup_uncharge_page() will do enough
  3837. * check. (see prepare_charge() also)
  3838. */
  3839. if (anon)
  3840. mem_cgroup_uncharge_page(used);
  3841. }
  3842. /*
  3843. * At replace page cache, newpage is not under any memcg but it's on
  3844. * LRU. So, this function doesn't touch res_counter but handles LRU
  3845. * in correct way. Both pages are locked so we cannot race with uncharge.
  3846. */
  3847. void mem_cgroup_replace_page_cache(struct page *oldpage,
  3848. struct page *newpage)
  3849. {
  3850. struct mem_cgroup *memcg = NULL;
  3851. struct page_cgroup *pc;
  3852. enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
  3853. if (mem_cgroup_disabled())
  3854. return;
  3855. pc = lookup_page_cgroup(oldpage);
  3856. /* fix accounting on old pages */
  3857. lock_page_cgroup(pc);
  3858. if (PageCgroupUsed(pc)) {
  3859. memcg = pc->mem_cgroup;
  3860. mem_cgroup_charge_statistics(memcg, false, -1);
  3861. ClearPageCgroupUsed(pc);
  3862. }
  3863. unlock_page_cgroup(pc);
  3864. /*
  3865. * When called from shmem_replace_page(), in some cases the
  3866. * oldpage has already been charged, and in some cases not.
  3867. */
  3868. if (!memcg)
  3869. return;
  3870. /*
  3871. * Even if newpage->mapping was NULL before starting replacement,
  3872. * the newpage may be on LRU(or pagevec for LRU) already. We lock
  3873. * LRU while we overwrite pc->mem_cgroup.
  3874. */
  3875. __mem_cgroup_commit_charge(memcg, newpage, 1, type, true);
  3876. }
  3877. #ifdef CONFIG_DEBUG_VM
  3878. static struct page_cgroup *lookup_page_cgroup_used(struct page *page)
  3879. {
  3880. struct page_cgroup *pc;
  3881. pc = lookup_page_cgroup(page);
  3882. /*
  3883. * Can be NULL while feeding pages into the page allocator for
  3884. * the first time, i.e. during boot or memory hotplug;
  3885. * or when mem_cgroup_disabled().
  3886. */
  3887. if (likely(pc) && PageCgroupUsed(pc))
  3888. return pc;
  3889. return NULL;
  3890. }
  3891. bool mem_cgroup_bad_page_check(struct page *page)
  3892. {
  3893. if (mem_cgroup_disabled())
  3894. return false;
  3895. return lookup_page_cgroup_used(page) != NULL;
  3896. }
  3897. void mem_cgroup_print_bad_page(struct page *page)
  3898. {
  3899. struct page_cgroup *pc;
  3900. pc = lookup_page_cgroup_used(page);
  3901. if (pc) {
  3902. pr_alert("pc:%p pc->flags:%lx pc->mem_cgroup:%p\n",
  3903. pc, pc->flags, pc->mem_cgroup);
  3904. }
  3905. }
  3906. #endif
  3907. static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
  3908. unsigned long long val)
  3909. {
  3910. int retry_count;
  3911. u64 memswlimit, memlimit;
  3912. int ret = 0;
  3913. int children = mem_cgroup_count_children(memcg);
  3914. u64 curusage, oldusage;
  3915. int enlarge;
  3916. /*
  3917. * For keeping hierarchical_reclaim simple, how long we should retry
  3918. * is depends on callers. We set our retry-count to be function
  3919. * of # of children which we should visit in this loop.
  3920. */
  3921. retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;
  3922. oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
  3923. enlarge = 0;
  3924. while (retry_count) {
  3925. if (signal_pending(current)) {
  3926. ret = -EINTR;
  3927. break;
  3928. }
  3929. /*
  3930. * Rather than hide all in some function, I do this in
  3931. * open coded manner. You see what this really does.
  3932. * We have to guarantee memcg->res.limit <= memcg->memsw.limit.
  3933. */
  3934. mutex_lock(&set_limit_mutex);
  3935. memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
  3936. if (memswlimit < val) {
  3937. ret = -EINVAL;
  3938. mutex_unlock(&set_limit_mutex);
  3939. break;
  3940. }
  3941. memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
  3942. if (memlimit < val)
  3943. enlarge = 1;
  3944. ret = res_counter_set_limit(&memcg->res, val);
  3945. if (!ret) {
  3946. if (memswlimit == val)
  3947. memcg->memsw_is_minimum = true;
  3948. else
  3949. memcg->memsw_is_minimum = false;
  3950. }
  3951. mutex_unlock(&set_limit_mutex);
  3952. if (!ret)
  3953. break;
  3954. mem_cgroup_reclaim(memcg, GFP_KERNEL,
  3955. MEM_CGROUP_RECLAIM_SHRINK);
  3956. curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
  3957. /* Usage is reduced ? */
  3958. if (curusage >= oldusage)
  3959. retry_count--;
  3960. else
  3961. oldusage = curusage;
  3962. }
  3963. if (!ret && enlarge)
  3964. memcg_oom_recover(memcg);
  3965. return ret;
  3966. }
  3967. static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
  3968. unsigned long long val)
  3969. {
  3970. int retry_count;
  3971. u64 memlimit, memswlimit, oldusage, curusage;
  3972. int children = mem_cgroup_count_children(memcg);
  3973. int ret = -EBUSY;
  3974. int enlarge = 0;
  3975. /* see mem_cgroup_resize_res_limit */
  3976. retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
  3977. oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
  3978. while (retry_count) {
  3979. if (signal_pending(current)) {
  3980. ret = -EINTR;
  3981. break;
  3982. }
  3983. /*
  3984. * Rather than hide all in some function, I do this in
  3985. * open coded manner. You see what this really does.
  3986. * We have to guarantee memcg->res.limit <= memcg->memsw.limit.
  3987. */
  3988. mutex_lock(&set_limit_mutex);
  3989. memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
  3990. if (memlimit > val) {
  3991. ret = -EINVAL;
  3992. mutex_unlock(&set_limit_mutex);
  3993. break;
  3994. }
  3995. memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
  3996. if (memswlimit < val)
  3997. enlarge = 1;
  3998. ret = res_counter_set_limit(&memcg->memsw, val);
  3999. if (!ret) {
  4000. if (memlimit == val)
  4001. memcg->memsw_is_minimum = true;
  4002. else
  4003. memcg->memsw_is_minimum = false;
  4004. }
  4005. mutex_unlock(&set_limit_mutex);
  4006. if (!ret)
  4007. break;
  4008. mem_cgroup_reclaim(memcg, GFP_KERNEL,
  4009. MEM_CGROUP_RECLAIM_NOSWAP |
  4010. MEM_CGROUP_RECLAIM_SHRINK);
  4011. curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
  4012. /* Usage is reduced ? */
  4013. if (curusage >= oldusage)
  4014. retry_count--;
  4015. else
  4016. oldusage = curusage;
  4017. }
  4018. if (!ret && enlarge)
  4019. memcg_oom_recover(memcg);
  4020. return ret;
  4021. }
  4022. unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
  4023. gfp_t gfp_mask,
  4024. unsigned long *total_scanned)
  4025. {
  4026. unsigned long nr_reclaimed = 0;
  4027. struct mem_cgroup_per_zone *mz, *next_mz = NULL;
  4028. unsigned long reclaimed;
  4029. int loop = 0;
  4030. struct mem_cgroup_tree_per_zone *mctz;
  4031. unsigned long long excess;
  4032. unsigned long nr_scanned;
  4033. if (order > 0)
  4034. return 0;
  4035. mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone));
  4036. /*
  4037. * This loop can run a while, specially if mem_cgroup's continuously
  4038. * keep exceeding their soft limit and putting the system under
  4039. * pressure
  4040. */
  4041. do {
  4042. if (next_mz)
  4043. mz = next_mz;
  4044. else
  4045. mz = mem_cgroup_largest_soft_limit_node(mctz);
  4046. if (!mz)
  4047. break;
  4048. nr_scanned = 0;
  4049. reclaimed = mem_cgroup_soft_reclaim(mz->memcg, zone,
  4050. gfp_mask, &nr_scanned);
  4051. nr_reclaimed += reclaimed;
  4052. *total_scanned += nr_scanned;
  4053. spin_lock(&mctz->lock);
  4054. /*
  4055. * If we failed to reclaim anything from this memory cgroup
  4056. * it is time to move on to the next cgroup
  4057. */
  4058. next_mz = NULL;
  4059. if (!reclaimed) {
  4060. do {
  4061. /*
  4062. * Loop until we find yet another one.
  4063. *
  4064. * By the time we get the soft_limit lock
  4065. * again, someone might have aded the
  4066. * group back on the RB tree. Iterate to
  4067. * make sure we get a different mem.
  4068. * mem_cgroup_largest_soft_limit_node returns
  4069. * NULL if no other cgroup is present on
  4070. * the tree
  4071. */
  4072. next_mz =
  4073. __mem_cgroup_largest_soft_limit_node(mctz);
  4074. if (next_mz == mz)
  4075. css_put(&next_mz->memcg->css);
  4076. else /* next_mz == NULL or other memcg */
  4077. break;
  4078. } while (1);
  4079. }
  4080. __mem_cgroup_remove_exceeded(mz->memcg, mz, mctz);
  4081. excess = res_counter_soft_limit_excess(&mz->memcg->res);
  4082. /*
  4083. * One school of thought says that we should not add
  4084. * back the node to the tree if reclaim returns 0.
  4085. * But our reclaim could return 0, simply because due
  4086. * to priority we are exposing a smaller subset of
  4087. * memory to reclaim from. Consider this as a longer
  4088. * term TODO.
  4089. */
  4090. /* If excess == 0, no tree ops */
  4091. __mem_cgroup_insert_exceeded(mz->memcg, mz, mctz, excess);
  4092. spin_unlock(&mctz->lock);
  4093. css_put(&mz->memcg->css);
  4094. loop++;
  4095. /*
  4096. * Could not reclaim anything and there are no more
  4097. * mem cgroups to try or we seem to be looping without
  4098. * reclaiming anything.
  4099. */
  4100. if (!nr_reclaimed &&
  4101. (next_mz == NULL ||
  4102. loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
  4103. break;
  4104. } while (!nr_reclaimed);
  4105. if (next_mz)
  4106. css_put(&next_mz->memcg->css);
  4107. return nr_reclaimed;
  4108. }
  4109. /**
  4110. * mem_cgroup_force_empty_list - clears LRU of a group
  4111. * @memcg: group to clear
  4112. * @node: NUMA node
  4113. * @zid: zone id
  4114. * @lru: lru to to clear
  4115. *
  4116. * Traverse a specified page_cgroup list and try to drop them all. This doesn't
  4117. * reclaim the pages page themselves - pages are moved to the parent (or root)
  4118. * group.
  4119. */
  4120. static void mem_cgroup_force_empty_list(struct mem_cgroup *memcg,
  4121. int node, int zid, enum lru_list lru)
  4122. {
  4123. struct lruvec *lruvec;
  4124. unsigned long flags;
  4125. struct list_head *list;
  4126. struct page *busy;
  4127. struct zone *zone;
  4128. zone = &NODE_DATA(node)->node_zones[zid];
  4129. lruvec = mem_cgroup_zone_lruvec(zone, memcg);
  4130. list = &lruvec->lists[lru];
  4131. busy = NULL;
  4132. do {
  4133. struct page_cgroup *pc;
  4134. struct page *page;
  4135. spin_lock_irqsave(&zone->lru_lock, flags);
  4136. if (list_empty(list)) {
  4137. spin_unlock_irqrestore(&zone->lru_lock, flags);
  4138. break;
  4139. }
  4140. page = list_entry(list->prev, struct page, lru);
  4141. if (busy == page) {
  4142. list_move(&page->lru, list);
  4143. busy = NULL;
  4144. spin_unlock_irqrestore(&zone->lru_lock, flags);
  4145. continue;
  4146. }
  4147. spin_unlock_irqrestore(&zone->lru_lock, flags);
  4148. pc = lookup_page_cgroup(page);
  4149. if (mem_cgroup_move_parent(page, pc, memcg)) {
  4150. /* found lock contention or "pc" is obsolete. */
  4151. busy = page;
  4152. cond_resched();
  4153. } else
  4154. busy = NULL;
  4155. } while (!list_empty(list));
  4156. }
  4157. /*
  4158. * make mem_cgroup's charge to be 0 if there is no task by moving
  4159. * all the charges and pages to the parent.
  4160. * This enables deleting this mem_cgroup.
  4161. *
  4162. * Caller is responsible for holding css reference on the memcg.
  4163. */
  4164. static void mem_cgroup_reparent_charges(struct mem_cgroup *memcg)
  4165. {
  4166. int node, zid;
  4167. u64 usage;
  4168. do {
  4169. /* This is for making all *used* pages to be on LRU. */
  4170. lru_add_drain_all();
  4171. drain_all_stock_sync(memcg);
  4172. mem_cgroup_start_move(memcg);
  4173. for_each_node_state(node, N_MEMORY) {
  4174. for (zid = 0; zid < MAX_NR_ZONES; zid++) {
  4175. enum lru_list lru;
  4176. for_each_lru(lru) {
  4177. mem_cgroup_force_empty_list(memcg,
  4178. node, zid, lru);
  4179. }
  4180. }
  4181. }
  4182. mem_cgroup_end_move(memcg);
  4183. memcg_oom_recover(memcg);
  4184. cond_resched();
  4185. /*
  4186. * Kernel memory may not necessarily be trackable to a specific
  4187. * process. So they are not migrated, and therefore we can't
  4188. * expect their value to drop to 0 here.
  4189. * Having res filled up with kmem only is enough.
  4190. *
  4191. * This is a safety check because mem_cgroup_force_empty_list
  4192. * could have raced with mem_cgroup_replace_page_cache callers
  4193. * so the lru seemed empty but the page could have been added
  4194. * right after the check. RES_USAGE should be safe as we always
  4195. * charge before adding to the LRU.
  4196. */
  4197. usage = res_counter_read_u64(&memcg->res, RES_USAGE) -
  4198. res_counter_read_u64(&memcg->kmem, RES_USAGE);
  4199. } while (usage > 0);
  4200. }
  4201. /*
  4202. * This mainly exists for tests during the setting of set of use_hierarchy.
  4203. * Since this is the very setting we are changing, the current hierarchy value
  4204. * is meaningless
  4205. */
  4206. static inline bool __memcg_has_children(struct mem_cgroup *memcg)
  4207. {
  4208. struct cgroup *pos;
  4209. /* bounce at first found */
  4210. cgroup_for_each_child(pos, memcg->css.cgroup)
  4211. return true;
  4212. return false;
  4213. }
  4214. /*
  4215. * Must be called with memcg_create_mutex held, unless the cgroup is guaranteed
  4216. * to be already dead (as in mem_cgroup_force_empty, for instance). This is
  4217. * from mem_cgroup_count_children(), in the sense that we don't really care how
  4218. * many children we have; we only need to know if we have any. It also counts
  4219. * any memcg without hierarchy as infertile.
  4220. */
  4221. static inline bool memcg_has_children(struct mem_cgroup *memcg)
  4222. {
  4223. return memcg->use_hierarchy && __memcg_has_children(memcg);
  4224. }
  4225. /*
  4226. * Reclaims as many pages from the given memcg as possible and moves
  4227. * the rest to the parent.
  4228. *
  4229. * Caller is responsible for holding css reference for memcg.
  4230. */
  4231. static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
  4232. {
  4233. int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
  4234. struct cgroup *cgrp = memcg->css.cgroup;
  4235. /* returns EBUSY if there is a task or if we come here twice. */
  4236. if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
  4237. return -EBUSY;
  4238. /* we call try-to-free pages for make this cgroup empty */
  4239. lru_add_drain_all();
  4240. /* try to free all pages in this cgroup */
  4241. while (nr_retries && res_counter_read_u64(&memcg->res, RES_USAGE) > 0) {
  4242. int progress;
  4243. if (signal_pending(current))
  4244. return -EINTR;
  4245. progress = try_to_free_mem_cgroup_pages(memcg, GFP_KERNEL,
  4246. false);
  4247. if (!progress) {
  4248. nr_retries--;
  4249. /* maybe some writeback is necessary */
  4250. congestion_wait(BLK_RW_ASYNC, HZ/10);
  4251. }
  4252. }
  4253. lru_add_drain();
  4254. mem_cgroup_reparent_charges(memcg);
  4255. return 0;
  4256. }
  4257. static int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event)
  4258. {
  4259. struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
  4260. int ret;
  4261. if (mem_cgroup_is_root(memcg))
  4262. return -EINVAL;
  4263. css_get(&memcg->css);
  4264. ret = mem_cgroup_force_empty(memcg);
  4265. css_put(&memcg->css);
  4266. return ret;
  4267. }
  4268. static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft)
  4269. {
  4270. return mem_cgroup_from_cont(cont)->use_hierarchy;
  4271. }
  4272. static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
  4273. u64 val)
  4274. {
  4275. int retval = 0;
  4276. struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
  4277. struct cgroup *parent = cont->parent;
  4278. struct mem_cgroup *parent_memcg = NULL;
  4279. if (parent)
  4280. parent_memcg = mem_cgroup_from_cont(parent);
  4281. mutex_lock(&memcg_create_mutex);
  4282. if (memcg->use_hierarchy == val)
  4283. goto out;
  4284. /*
  4285. * If parent's use_hierarchy is set, we can't make any modifications
  4286. * in the child subtrees. If it is unset, then the change can
  4287. * occur, provided the current cgroup has no children.
  4288. *
  4289. * For the root cgroup, parent_mem is NULL, we allow value to be
  4290. * set if there are no children.
  4291. */
  4292. if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
  4293. (val == 1 || val == 0)) {
  4294. if (!__memcg_has_children(memcg))
  4295. memcg->use_hierarchy = val;
  4296. else
  4297. retval = -EBUSY;
  4298. } else
  4299. retval = -EINVAL;
  4300. out:
  4301. mutex_unlock(&memcg_create_mutex);
  4302. return retval;
  4303. }
  4304. static unsigned long mem_cgroup_recursive_stat(struct mem_cgroup *memcg,
  4305. enum mem_cgroup_stat_index idx)
  4306. {
  4307. struct mem_cgroup *iter;
  4308. long val = 0;
  4309. /* Per-cpu values can be negative, use a signed accumulator */
  4310. for_each_mem_cgroup_tree(iter, memcg)
  4311. val += mem_cgroup_read_stat(iter, idx);
  4312. if (val < 0) /* race ? */
  4313. val = 0;
  4314. return val;
  4315. }
  4316. static inline u64 mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
  4317. {
  4318. u64 val;
  4319. if (!mem_cgroup_is_root(memcg)) {
  4320. if (!swap)
  4321. return res_counter_read_u64(&memcg->res, RES_USAGE);
  4322. else
  4323. return res_counter_read_u64(&memcg->memsw, RES_USAGE);
  4324. }
  4325. val = mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_CACHE);
  4326. val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_RSS);
  4327. if (swap)
  4328. val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_SWAP);
  4329. return val << PAGE_SHIFT;
  4330. }
  4331. static ssize_t mem_cgroup_read(struct cgroup *cont, struct cftype *cft,
  4332. struct file *file, char __user *buf,
  4333. size_t nbytes, loff_t *ppos)
  4334. {
  4335. struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
  4336. char str[64];
  4337. u64 val;
  4338. int name, len;
  4339. enum res_type type;
  4340. type = MEMFILE_TYPE(cft->private);
  4341. name = MEMFILE_ATTR(cft->private);
  4342. if (!do_swap_account && type == _MEMSWAP)
  4343. return -EOPNOTSUPP;
  4344. switch (type) {
  4345. case _MEM:
  4346. if (name == RES_USAGE)
  4347. val = mem_cgroup_usage(memcg, false);
  4348. else
  4349. val = res_counter_read_u64(&memcg->res, name);
  4350. break;
  4351. case _MEMSWAP:
  4352. if (name == RES_USAGE)
  4353. val = mem_cgroup_usage(memcg, true);
  4354. else
  4355. val = res_counter_read_u64(&memcg->memsw, name);
  4356. break;
  4357. case _KMEM:
  4358. val = res_counter_read_u64(&memcg->kmem, name);
  4359. break;
  4360. default:
  4361. BUG();
  4362. }
  4363. len = scnprintf(str, sizeof(str), "%llu\n", (unsigned long long)val);
  4364. return simple_read_from_buffer(buf, nbytes, ppos, str, len);
  4365. }
  4366. static int memcg_update_kmem_limit(struct cgroup *cont, u64 val)
  4367. {
  4368. int ret = -EINVAL;
  4369. #ifdef CONFIG_MEMCG_KMEM
  4370. struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
  4371. /*
  4372. * For simplicity, we won't allow this to be disabled. It also can't
  4373. * be changed if the cgroup has children already, or if tasks had
  4374. * already joined.
  4375. *
  4376. * If tasks join before we set the limit, a person looking at
  4377. * kmem.usage_in_bytes will have no way to determine when it took
  4378. * place, which makes the value quite meaningless.
  4379. *
  4380. * After it first became limited, changes in the value of the limit are
  4381. * of course permitted.
  4382. */
  4383. mutex_lock(&memcg_create_mutex);
  4384. mutex_lock(&set_limit_mutex);
  4385. if (!memcg->kmem_account_flags && val != RESOURCE_MAX) {
  4386. if (cgroup_task_count(cont) || memcg_has_children(memcg)) {
  4387. ret = -EBUSY;
  4388. goto out;
  4389. }
  4390. ret = res_counter_set_limit(&memcg->kmem, val);
  4391. VM_BUG_ON(ret);
  4392. ret = memcg_update_cache_sizes(memcg);
  4393. if (ret) {
  4394. res_counter_set_limit(&memcg->kmem, RESOURCE_MAX);
  4395. goto out;
  4396. }
  4397. static_key_slow_inc(&memcg_kmem_enabled_key);
  4398. /*
  4399. * setting the active bit after the inc will guarantee no one
  4400. * starts accounting before all call sites are patched
  4401. */
  4402. memcg_kmem_set_active(memcg);
  4403. /*
  4404. * kmem charges can outlive the cgroup. In the case of slab
  4405. * pages, for instance, a page contain objects from various
  4406. * processes, so it is unfeasible to migrate them away. We
  4407. * need to reference count the memcg because of that.
  4408. */
  4409. mem_cgroup_get(memcg);
  4410. } else
  4411. ret = res_counter_set_limit(&memcg->kmem, val);
  4412. out:
  4413. mutex_unlock(&set_limit_mutex);
  4414. mutex_unlock(&memcg_create_mutex);
  4415. #endif
  4416. return ret;
  4417. }
  4418. #ifdef CONFIG_MEMCG_KMEM
  4419. static int memcg_propagate_kmem(struct mem_cgroup *memcg)
  4420. {
  4421. int ret = 0;
  4422. struct mem_cgroup *parent = parent_mem_cgroup(memcg);
  4423. if (!parent)
  4424. goto out;
  4425. memcg->kmem_account_flags = parent->kmem_account_flags;
  4426. /*
  4427. * When that happen, we need to disable the static branch only on those
  4428. * memcgs that enabled it. To achieve this, we would be forced to
  4429. * complicate the code by keeping track of which memcgs were the ones
  4430. * that actually enabled limits, and which ones got it from its
  4431. * parents.
  4432. *
  4433. * It is a lot simpler just to do static_key_slow_inc() on every child
  4434. * that is accounted.
  4435. */
  4436. if (!memcg_kmem_is_active(memcg))
  4437. goto out;
  4438. /*
  4439. * destroy(), called if we fail, will issue static_key_slow_inc() and
  4440. * mem_cgroup_put() if kmem is enabled. We have to either call them
  4441. * unconditionally, or clear the KMEM_ACTIVE flag. I personally find
  4442. * this more consistent, since it always leads to the same destroy path
  4443. */
  4444. mem_cgroup_get(memcg);
  4445. static_key_slow_inc(&memcg_kmem_enabled_key);
  4446. mutex_lock(&set_limit_mutex);
  4447. ret = memcg_update_cache_sizes(memcg);
  4448. mutex_unlock(&set_limit_mutex);
  4449. out:
  4450. return ret;
  4451. }
  4452. #endif /* CONFIG_MEMCG_KMEM */
  4453. /*
  4454. * The user of this function is...
  4455. * RES_LIMIT.
  4456. */
  4457. static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
  4458. const char *buffer)
  4459. {
  4460. struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
  4461. enum res_type type;
  4462. int name;
  4463. unsigned long long val;
  4464. int ret;
  4465. type = MEMFILE_TYPE(cft->private);
  4466. name = MEMFILE_ATTR(cft->private);
  4467. if (!do_swap_account && type == _MEMSWAP)
  4468. return -EOPNOTSUPP;
  4469. switch (name) {
  4470. case RES_LIMIT:
  4471. if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
  4472. ret = -EINVAL;
  4473. break;
  4474. }
  4475. /* This function does all necessary parse...reuse it */
  4476. ret = res_counter_memparse_write_strategy(buffer, &val);
  4477. if (ret)
  4478. break;
  4479. if (type == _MEM)
  4480. ret = mem_cgroup_resize_limit(memcg, val);
  4481. else if (type == _MEMSWAP)
  4482. ret = mem_cgroup_resize_memsw_limit(memcg, val);
  4483. else if (type == _KMEM)
  4484. ret = memcg_update_kmem_limit(cont, val);
  4485. else
  4486. return -EINVAL;
  4487. break;
  4488. case RES_SOFT_LIMIT:
  4489. ret = res_counter_memparse_write_strategy(buffer, &val);
  4490. if (ret)
  4491. break;
  4492. /*
  4493. * For memsw, soft limits are hard to implement in terms
  4494. * of semantics, for now, we support soft limits for
  4495. * control without swap
  4496. */
  4497. if (type == _MEM)
  4498. ret = res_counter_set_soft_limit(&memcg->res, val);
  4499. else
  4500. ret = -EINVAL;
  4501. break;
  4502. default:
  4503. ret = -EINVAL; /* should be BUG() ? */
  4504. break;
  4505. }
  4506. return ret;
  4507. }
  4508. static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg,
  4509. unsigned long long *mem_limit, unsigned long long *memsw_limit)
  4510. {
  4511. struct cgroup *cgroup;
  4512. unsigned long long min_limit, min_memsw_limit, tmp;
  4513. min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
  4514. min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
  4515. cgroup = memcg->css.cgroup;
  4516. if (!memcg->use_hierarchy)
  4517. goto out;
  4518. while (cgroup->parent) {
  4519. cgroup = cgroup->parent;
  4520. memcg = mem_cgroup_from_cont(cgroup);
  4521. if (!memcg->use_hierarchy)
  4522. break;
  4523. tmp = res_counter_read_u64(&memcg->res, RES_LIMIT);
  4524. min_limit = min(min_limit, tmp);
  4525. tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
  4526. min_memsw_limit = min(min_memsw_limit, tmp);
  4527. }
  4528. out:
  4529. *mem_limit = min_limit;
  4530. *memsw_limit = min_memsw_limit;
  4531. }
  4532. static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
  4533. {
  4534. struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
  4535. int name;
  4536. enum res_type type;
  4537. type = MEMFILE_TYPE(event);
  4538. name = MEMFILE_ATTR(event);
  4539. if (!do_swap_account && type == _MEMSWAP)
  4540. return -EOPNOTSUPP;
  4541. switch (name) {
  4542. case RES_MAX_USAGE:
  4543. if (type == _MEM)
  4544. res_counter_reset_max(&memcg->res);
  4545. else if (type == _MEMSWAP)
  4546. res_counter_reset_max(&memcg->memsw);
  4547. else if (type == _KMEM)
  4548. res_counter_reset_max(&memcg->kmem);
  4549. else
  4550. return -EINVAL;
  4551. break;
  4552. case RES_FAILCNT:
  4553. if (type == _MEM)
  4554. res_counter_reset_failcnt(&memcg->res);
  4555. else if (type == _MEMSWAP)
  4556. res_counter_reset_failcnt(&memcg->memsw);
  4557. else if (type == _KMEM)
  4558. res_counter_reset_failcnt(&memcg->kmem);
  4559. else
  4560. return -EINVAL;
  4561. break;
  4562. }
  4563. return 0;
  4564. }
  4565. static u64 mem_cgroup_move_charge_read(struct cgroup *cgrp,
  4566. struct cftype *cft)
  4567. {
  4568. return mem_cgroup_from_cont(cgrp)->move_charge_at_immigrate;
  4569. }
  4570. #ifdef CONFIG_MMU
  4571. static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
  4572. struct cftype *cft, u64 val)
  4573. {
  4574. struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
  4575. if (val >= (1 << NR_MOVE_TYPE))
  4576. return -EINVAL;
  4577. /*
  4578. * No kind of locking is needed in here, because ->can_attach() will
  4579. * check this value once in the beginning of the process, and then carry
  4580. * on with stale data. This means that changes to this value will only
  4581. * affect task migrations starting after the change.
  4582. */
  4583. memcg->move_charge_at_immigrate = val;
  4584. return 0;
  4585. }
  4586. #else
  4587. static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
  4588. struct cftype *cft, u64 val)
  4589. {
  4590. return -ENOSYS;
  4591. }
  4592. #endif
  4593. #ifdef CONFIG_NUMA
  4594. static int memcg_numa_stat_show(struct cgroup *cont, struct cftype *cft,
  4595. struct seq_file *m)
  4596. {
  4597. int nid;
  4598. unsigned long total_nr, file_nr, anon_nr, unevictable_nr;
  4599. unsigned long node_nr;
  4600. struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
  4601. total_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL);
  4602. seq_printf(m, "total=%lu", total_nr);
  4603. for_each_node_state(nid, N_MEMORY) {
  4604. node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL);
  4605. seq_printf(m, " N%d=%lu", nid, node_nr);
  4606. }
  4607. seq_putc(m, '\n');
  4608. file_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL_FILE);
  4609. seq_printf(m, "file=%lu", file_nr);
  4610. for_each_node_state(nid, N_MEMORY) {
  4611. node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
  4612. LRU_ALL_FILE);
  4613. seq_printf(m, " N%d=%lu", nid, node_nr);
  4614. }
  4615. seq_putc(m, '\n');
  4616. anon_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL_ANON);
  4617. seq_printf(m, "anon=%lu", anon_nr);
  4618. for_each_node_state(nid, N_MEMORY) {
  4619. node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
  4620. LRU_ALL_ANON);
  4621. seq_printf(m, " N%d=%lu", nid, node_nr);
  4622. }
  4623. seq_putc(m, '\n');
  4624. unevictable_nr = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_UNEVICTABLE));
  4625. seq_printf(m, "unevictable=%lu", unevictable_nr);
  4626. for_each_node_state(nid, N_MEMORY) {
  4627. node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
  4628. BIT(LRU_UNEVICTABLE));
  4629. seq_printf(m, " N%d=%lu", nid, node_nr);
  4630. }
  4631. seq_putc(m, '\n');
  4632. return 0;
  4633. }
  4634. #endif /* CONFIG_NUMA */
  4635. static inline void mem_cgroup_lru_names_not_uptodate(void)
  4636. {
  4637. BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);
  4638. }
  4639. static int memcg_stat_show(struct cgroup *cont, struct cftype *cft,
  4640. struct seq_file *m)
  4641. {
  4642. struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
  4643. struct mem_cgroup *mi;
  4644. unsigned int i;
  4645. for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
  4646. if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
  4647. continue;
  4648. seq_printf(m, "%s %ld\n", mem_cgroup_stat_names[i],
  4649. mem_cgroup_read_stat(memcg, i) * PAGE_SIZE);
  4650. }
  4651. for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++)
  4652. seq_printf(m, "%s %lu\n", mem_cgroup_events_names[i],
  4653. mem_cgroup_read_events(memcg, i));
  4654. for (i = 0; i < NR_LRU_LISTS; i++)
  4655. seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
  4656. mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);
  4657. /* Hierarchical information */
  4658. {
  4659. unsigned long long limit, memsw_limit;
  4660. memcg_get_hierarchical_limit(memcg, &limit, &memsw_limit);
  4661. seq_printf(m, "hierarchical_memory_limit %llu\n", limit);
  4662. if (do_swap_account)
  4663. seq_printf(m, "hierarchical_memsw_limit %llu\n",
  4664. memsw_limit);
  4665. }
  4666. for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
  4667. long long val = 0;
  4668. if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
  4669. continue;
  4670. for_each_mem_cgroup_tree(mi, memcg)
  4671. val += mem_cgroup_read_stat(mi, i) * PAGE_SIZE;
  4672. seq_printf(m, "total_%s %lld\n", mem_cgroup_stat_names[i], val);
  4673. }
  4674. for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
  4675. unsigned long long val = 0;
  4676. for_each_mem_cgroup_tree(mi, memcg)
  4677. val += mem_cgroup_read_events(mi, i);
  4678. seq_printf(m, "total_%s %llu\n",
  4679. mem_cgroup_events_names[i], val);
  4680. }
  4681. for (i = 0; i < NR_LRU_LISTS; i++) {
  4682. unsigned long long val = 0;
  4683. for_each_mem_cgroup_tree(mi, memcg)
  4684. val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE;
  4685. seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val);
  4686. }
  4687. #ifdef CONFIG_DEBUG_VM
  4688. {
  4689. int nid, zid;
  4690. struct mem_cgroup_per_zone *mz;
  4691. struct zone_reclaim_stat *rstat;
  4692. unsigned long recent_rotated[2] = {0, 0};
  4693. unsigned long recent_scanned[2] = {0, 0};
  4694. for_each_online_node(nid)
  4695. for (zid = 0; zid < MAX_NR_ZONES; zid++) {
  4696. mz = mem_cgroup_zoneinfo(memcg, nid, zid);
  4697. rstat = &mz->lruvec.reclaim_stat;
  4698. recent_rotated[0] += rstat->recent_rotated[0];
  4699. recent_rotated[1] += rstat->recent_rotated[1];
  4700. recent_scanned[0] += rstat->recent_scanned[0];
  4701. recent_scanned[1] += rstat->recent_scanned[1];
  4702. }
  4703. seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
  4704. seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
  4705. seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
  4706. seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
  4707. }
  4708. #endif
  4709. return 0;
  4710. }
  4711. static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft)
  4712. {
  4713. struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
  4714. return mem_cgroup_swappiness(memcg);
  4715. }
  4716. static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft,
  4717. u64 val)
  4718. {
  4719. struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
  4720. struct mem_cgroup *parent;
  4721. if (val > 100)
  4722. return -EINVAL;
  4723. if (cgrp->parent == NULL)
  4724. return -EINVAL;
  4725. parent = mem_cgroup_from_cont(cgrp->parent);
  4726. mutex_lock(&memcg_create_mutex);
  4727. /* If under hierarchy, only empty-root can set this value */
  4728. if ((parent->use_hierarchy) || memcg_has_children(memcg)) {
  4729. mutex_unlock(&memcg_create_mutex);
  4730. return -EINVAL;
  4731. }
  4732. memcg->swappiness = val;
  4733. mutex_unlock(&memcg_create_mutex);
  4734. return 0;
  4735. }
  4736. static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
  4737. {
  4738. struct mem_cgroup_threshold_ary *t;
  4739. u64 usage;
  4740. int i;
  4741. rcu_read_lock();
  4742. if (!swap)
  4743. t = rcu_dereference(memcg->thresholds.primary);
  4744. else
  4745. t = rcu_dereference(memcg->memsw_thresholds.primary);
  4746. if (!t)
  4747. goto unlock;
  4748. usage = mem_cgroup_usage(memcg, swap);
  4749. /*
  4750. * current_threshold points to threshold just below or equal to usage.
  4751. * If it's not true, a threshold was crossed after last
  4752. * call of __mem_cgroup_threshold().
  4753. */
  4754. i = t->current_threshold;
  4755. /*
  4756. * Iterate backward over array of thresholds starting from
  4757. * current_threshold and check if a threshold is crossed.
  4758. * If none of thresholds below usage is crossed, we read
  4759. * only one element of the array here.
  4760. */
  4761. for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
  4762. eventfd_signal(t->entries[i].eventfd, 1);
  4763. /* i = current_threshold + 1 */
  4764. i++;
  4765. /*
  4766. * Iterate forward over array of thresholds starting from
  4767. * current_threshold+1 and check if a threshold is crossed.
  4768. * If none of thresholds above usage is crossed, we read
  4769. * only one element of the array here.
  4770. */
  4771. for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
  4772. eventfd_signal(t->entries[i].eventfd, 1);
  4773. /* Update current_threshold */
  4774. t->current_threshold = i - 1;
  4775. unlock:
  4776. rcu_read_unlock();
  4777. }
  4778. static void mem_cgroup_threshold(struct mem_cgroup *memcg)
  4779. {
  4780. while (memcg) {
  4781. __mem_cgroup_threshold(memcg, false);
  4782. if (do_swap_account)
  4783. __mem_cgroup_threshold(memcg, true);
  4784. memcg = parent_mem_cgroup(memcg);
  4785. }
  4786. }
  4787. static int compare_thresholds(const void *a, const void *b)
  4788. {
  4789. const struct mem_cgroup_threshold *_a = a;
  4790. const struct mem_cgroup_threshold *_b = b;
  4791. return _a->threshold - _b->threshold;
  4792. }
  4793. static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
  4794. {
  4795. struct mem_cgroup_eventfd_list *ev;
  4796. list_for_each_entry(ev, &memcg->oom_notify, list)
  4797. eventfd_signal(ev->eventfd, 1);
  4798. return 0;
  4799. }
  4800. static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
  4801. {
  4802. struct mem_cgroup *iter;
  4803. for_each_mem_cgroup_tree(iter, memcg)
  4804. mem_cgroup_oom_notify_cb(iter);
  4805. }
  4806. static int mem_cgroup_usage_register_event(struct cgroup *cgrp,
  4807. struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
  4808. {
  4809. struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
  4810. struct mem_cgroup_thresholds *thresholds;
  4811. struct mem_cgroup_threshold_ary *new;
  4812. enum res_type type = MEMFILE_TYPE(cft->private);
  4813. u64 threshold, usage;
  4814. int i, size, ret;
  4815. ret = res_counter_memparse_write_strategy(args, &threshold);
  4816. if (ret)
  4817. return ret;
  4818. mutex_lock(&memcg->thresholds_lock);
  4819. if (type == _MEM)
  4820. thresholds = &memcg->thresholds;
  4821. else if (type == _MEMSWAP)
  4822. thresholds = &memcg->memsw_thresholds;
  4823. else
  4824. BUG();
  4825. usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
  4826. /* Check if a threshold crossed before adding a new one */
  4827. if (thresholds->primary)
  4828. __mem_cgroup_threshold(memcg, type == _MEMSWAP);
  4829. size = thresholds->primary ? thresholds->primary->size + 1 : 1;
  4830. /* Allocate memory for new array of thresholds */
  4831. new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
  4832. GFP_KERNEL);
  4833. if (!new) {
  4834. ret = -ENOMEM;
  4835. goto unlock;
  4836. }
  4837. new->size = size;
  4838. /* Copy thresholds (if any) to new array */
  4839. if (thresholds->primary) {
  4840. memcpy(new->entries, thresholds->primary->entries, (size - 1) *
  4841. sizeof(struct mem_cgroup_threshold));
  4842. }
  4843. /* Add new threshold */
  4844. new->entries[size - 1].eventfd = eventfd;
  4845. new->entries[size - 1].threshold = threshold;
  4846. /* Sort thresholds. Registering of new threshold isn't time-critical */
  4847. sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
  4848. compare_thresholds, NULL);
  4849. /* Find current threshold */
  4850. new->current_threshold = -1;
  4851. for (i = 0; i < size; i++) {
  4852. if (new->entries[i].threshold <= usage) {
  4853. /*
  4854. * new->current_threshold will not be used until
  4855. * rcu_assign_pointer(), so it's safe to increment
  4856. * it here.
  4857. */
  4858. ++new->current_threshold;
  4859. } else
  4860. break;
  4861. }
  4862. /* Free old spare buffer and save old primary buffer as spare */
  4863. kfree(thresholds->spare);
  4864. thresholds->spare = thresholds->primary;
  4865. rcu_assign_pointer(thresholds->primary, new);
  4866. /* To be sure that nobody uses thresholds */
  4867. synchronize_rcu();
  4868. unlock:
  4869. mutex_unlock(&memcg->thresholds_lock);
  4870. return ret;
  4871. }
  4872. static void mem_cgroup_usage_unregister_event(struct cgroup *cgrp,
  4873. struct cftype *cft, struct eventfd_ctx *eventfd)
  4874. {
  4875. struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
  4876. struct mem_cgroup_thresholds *thresholds;
  4877. struct mem_cgroup_threshold_ary *new;
  4878. enum res_type type = MEMFILE_TYPE(cft->private);
  4879. u64 usage;
  4880. int i, j, size;
  4881. mutex_lock(&memcg->thresholds_lock);
  4882. if (type == _MEM)
  4883. thresholds = &memcg->thresholds;
  4884. else if (type == _MEMSWAP)
  4885. thresholds = &memcg->memsw_thresholds;
  4886. else
  4887. BUG();
  4888. if (!thresholds->primary)
  4889. goto unlock;
  4890. usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
  4891. /* Check if a threshold crossed before removing */
  4892. __mem_cgroup_threshold(memcg, type == _MEMSWAP);
  4893. /* Calculate new number of threshold */
  4894. size = 0;
  4895. for (i = 0; i < thresholds->primary->size; i++) {
  4896. if (thresholds->primary->entries[i].eventfd != eventfd)
  4897. size++;
  4898. }
  4899. new = thresholds->spare;
  4900. /* Set thresholds array to NULL if we don't have thresholds */
  4901. if (!size) {
  4902. kfree(new);
  4903. new = NULL;
  4904. goto swap_buffers;
  4905. }
  4906. new->size = size;
  4907. /* Copy thresholds and find current threshold */
  4908. new->current_threshold = -1;
  4909. for (i = 0, j = 0; i < thresholds->primary->size; i++) {
  4910. if (thresholds->primary->entries[i].eventfd == eventfd)
  4911. continue;
  4912. new->entries[j] = thresholds->primary->entries[i];
  4913. if (new->entries[j].threshold <= usage) {
  4914. /*
  4915. * new->current_threshold will not be used
  4916. * until rcu_assign_pointer(), so it's safe to increment
  4917. * it here.
  4918. */
  4919. ++new->current_threshold;
  4920. }
  4921. j++;
  4922. }
  4923. swap_buffers:
  4924. /* Swap primary and spare array */
  4925. thresholds->spare = thresholds->primary;
  4926. /* If all events are unregistered, free the spare array */
  4927. if (!new) {
  4928. kfree(thresholds->spare);
  4929. thresholds->spare = NULL;
  4930. }
  4931. rcu_assign_pointer(thresholds->primary, new);
  4932. /* To be sure that nobody uses thresholds */
  4933. synchronize_rcu();
  4934. unlock:
  4935. mutex_unlock(&memcg->thresholds_lock);
  4936. }
  4937. static int mem_cgroup_oom_register_event(struct cgroup *cgrp,
  4938. struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
  4939. {
  4940. struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
  4941. struct mem_cgroup_eventfd_list *event;
  4942. enum res_type type = MEMFILE_TYPE(cft->private);
  4943. BUG_ON(type != _OOM_TYPE);
  4944. event = kmalloc(sizeof(*event), GFP_KERNEL);
  4945. if (!event)
  4946. return -ENOMEM;
  4947. spin_lock(&memcg_oom_lock);
  4948. event->eventfd = eventfd;
  4949. list_add(&event->list, &memcg->oom_notify);
  4950. /* already in OOM ? */
  4951. if (atomic_read(&memcg->under_oom))
  4952. eventfd_signal(eventfd, 1);
  4953. spin_unlock(&memcg_oom_lock);
  4954. return 0;
  4955. }
  4956. static void mem_cgroup_oom_unregister_event(struct cgroup *cgrp,
  4957. struct cftype *cft, struct eventfd_ctx *eventfd)
  4958. {
  4959. struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
  4960. struct mem_cgroup_eventfd_list *ev, *tmp;
  4961. enum res_type type = MEMFILE_TYPE(cft->private);
  4962. BUG_ON(type != _OOM_TYPE);
  4963. spin_lock(&memcg_oom_lock);
  4964. list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
  4965. if (ev->eventfd == eventfd) {
  4966. list_del(&ev->list);
  4967. kfree(ev);
  4968. }
  4969. }
  4970. spin_unlock(&memcg_oom_lock);
  4971. }
  4972. static int mem_cgroup_oom_control_read(struct cgroup *cgrp,
  4973. struct cftype *cft, struct cgroup_map_cb *cb)
  4974. {
  4975. struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
  4976. cb->fill(cb, "oom_kill_disable", memcg->oom_kill_disable);
  4977. if (atomic_read(&memcg->under_oom))
  4978. cb->fill(cb, "under_oom", 1);
  4979. else
  4980. cb->fill(cb, "under_oom", 0);
  4981. return 0;
  4982. }
  4983. static int mem_cgroup_oom_control_write(struct cgroup *cgrp,
  4984. struct cftype *cft, u64 val)
  4985. {
  4986. struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
  4987. struct mem_cgroup *parent;
  4988. /* cannot set to root cgroup and only 0 and 1 are allowed */
  4989. if (!cgrp->parent || !((val == 0) || (val == 1)))
  4990. return -EINVAL;
  4991. parent = mem_cgroup_from_cont(cgrp->parent);
  4992. mutex_lock(&memcg_create_mutex);
  4993. /* oom-kill-disable is a flag for subhierarchy. */
  4994. if ((parent->use_hierarchy) || memcg_has_children(memcg)) {
  4995. mutex_unlock(&memcg_create_mutex);
  4996. return -EINVAL;
  4997. }
  4998. memcg->oom_kill_disable = val;
  4999. if (!val)
  5000. memcg_oom_recover(memcg);
  5001. mutex_unlock(&memcg_create_mutex);
  5002. return 0;
  5003. }
  5004. #ifdef CONFIG_MEMCG_KMEM
  5005. static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
  5006. {
  5007. int ret;
  5008. memcg->kmemcg_id = -1;
  5009. ret = memcg_propagate_kmem(memcg);
  5010. if (ret)
  5011. return ret;
  5012. return mem_cgroup_sockets_init(memcg, ss);
  5013. };
  5014. static void kmem_cgroup_destroy(struct mem_cgroup *memcg)
  5015. {
  5016. mem_cgroup_sockets_destroy(memcg);
  5017. memcg_kmem_mark_dead(memcg);
  5018. if (res_counter_read_u64(&memcg->kmem, RES_USAGE) != 0)
  5019. return;
  5020. /*
  5021. * Charges already down to 0, undo mem_cgroup_get() done in the charge
  5022. * path here, being careful not to race with memcg_uncharge_kmem: it is
  5023. * possible that the charges went down to 0 between mark_dead and the
  5024. * res_counter read, so in that case, we don't need the put
  5025. */
  5026. if (memcg_kmem_test_and_clear_dead(memcg))
  5027. mem_cgroup_put(memcg);
  5028. }
  5029. #else
  5030. static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
  5031. {
  5032. return 0;
  5033. }
  5034. static void kmem_cgroup_destroy(struct mem_cgroup *memcg)
  5035. {
  5036. }
  5037. #endif
  5038. static struct cftype mem_cgroup_files[] = {
  5039. {
  5040. .name = "usage_in_bytes",
  5041. .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
  5042. .read = mem_cgroup_read,
  5043. .register_event = mem_cgroup_usage_register_event,
  5044. .unregister_event = mem_cgroup_usage_unregister_event,
  5045. },
  5046. {
  5047. .name = "max_usage_in_bytes",
  5048. .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
  5049. .trigger = mem_cgroup_reset,
  5050. .read = mem_cgroup_read,
  5051. },
  5052. {
  5053. .name = "limit_in_bytes",
  5054. .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
  5055. .write_string = mem_cgroup_write,
  5056. .read = mem_cgroup_read,
  5057. },
  5058. {
  5059. .name = "soft_limit_in_bytes",
  5060. .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
  5061. .write_string = mem_cgroup_write,
  5062. .read = mem_cgroup_read,
  5063. },
  5064. {
  5065. .name = "failcnt",
  5066. .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
  5067. .trigger = mem_cgroup_reset,
  5068. .read = mem_cgroup_read,
  5069. },
  5070. {
  5071. .name = "stat",
  5072. .read_seq_string = memcg_stat_show,
  5073. },
  5074. {
  5075. .name = "force_empty",
  5076. .trigger = mem_cgroup_force_empty_write,
  5077. },
  5078. {
  5079. .name = "use_hierarchy",
  5080. .write_u64 = mem_cgroup_hierarchy_write,
  5081. .read_u64 = mem_cgroup_hierarchy_read,
  5082. },
  5083. {
  5084. .name = "swappiness",
  5085. .read_u64 = mem_cgroup_swappiness_read,
  5086. .write_u64 = mem_cgroup_swappiness_write,
  5087. },
  5088. {
  5089. .name = "move_charge_at_immigrate",
  5090. .read_u64 = mem_cgroup_move_charge_read,
  5091. .write_u64 = mem_cgroup_move_charge_write,
  5092. },
  5093. {
  5094. .name = "oom_control",
  5095. .read_map = mem_cgroup_oom_control_read,
  5096. .write_u64 = mem_cgroup_oom_control_write,
  5097. .register_event = mem_cgroup_oom_register_event,
  5098. .unregister_event = mem_cgroup_oom_unregister_event,
  5099. .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
  5100. },
  5101. #ifdef CONFIG_NUMA
  5102. {
  5103. .name = "numa_stat",
  5104. .read_seq_string = memcg_numa_stat_show,
  5105. },
  5106. #endif
  5107. #ifdef CONFIG_MEMCG_KMEM
  5108. {
  5109. .name = "kmem.limit_in_bytes",
  5110. .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
  5111. .write_string = mem_cgroup_write,
  5112. .read = mem_cgroup_read,
  5113. },
  5114. {
  5115. .name = "kmem.usage_in_bytes",
  5116. .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
  5117. .read = mem_cgroup_read,
  5118. },
  5119. {
  5120. .name = "kmem.failcnt",
  5121. .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
  5122. .trigger = mem_cgroup_reset,
  5123. .read = mem_cgroup_read,
  5124. },
  5125. {
  5126. .name = "kmem.max_usage_in_bytes",
  5127. .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
  5128. .trigger = mem_cgroup_reset,
  5129. .read = mem_cgroup_read,
  5130. },
  5131. #ifdef CONFIG_SLABINFO
  5132. {
  5133. .name = "kmem.slabinfo",
  5134. .read_seq_string = mem_cgroup_slabinfo_read,
  5135. },
  5136. #endif
  5137. #endif
  5138. { }, /* terminate */
  5139. };
  5140. #ifdef CONFIG_MEMCG_SWAP
  5141. static struct cftype memsw_cgroup_files[] = {
  5142. {
  5143. .name = "memsw.usage_in_bytes",
  5144. .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
  5145. .read = mem_cgroup_read,
  5146. .register_event = mem_cgroup_usage_register_event,
  5147. .unregister_event = mem_cgroup_usage_unregister_event,
  5148. },
  5149. {
  5150. .name = "memsw.max_usage_in_bytes",
  5151. .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
  5152. .trigger = mem_cgroup_reset,
  5153. .read = mem_cgroup_read,
  5154. },
  5155. {
  5156. .name = "memsw.limit_in_bytes",
  5157. .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
  5158. .write_string = mem_cgroup_write,
  5159. .read = mem_cgroup_read,
  5160. },
  5161. {
  5162. .name = "memsw.failcnt",
  5163. .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
  5164. .trigger = mem_cgroup_reset,
  5165. .read = mem_cgroup_read,
  5166. },
  5167. { }, /* terminate */
  5168. };
  5169. #endif
  5170. static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
  5171. {
  5172. struct mem_cgroup_per_node *pn;
  5173. struct mem_cgroup_per_zone *mz;
  5174. int zone, tmp = node;
  5175. /*
  5176. * This routine is called against possible nodes.
  5177. * But it's BUG to call kmalloc() against offline node.
  5178. *
  5179. * TODO: this routine can waste much memory for nodes which will
  5180. * never be onlined. It's better to use memory hotplug callback
  5181. * function.
  5182. */
  5183. if (!node_state(node, N_NORMAL_MEMORY))
  5184. tmp = -1;
  5185. pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
  5186. if (!pn)
  5187. return 1;
  5188. for (zone = 0; zone < MAX_NR_ZONES; zone++) {
  5189. mz = &pn->zoneinfo[zone];
  5190. lruvec_init(&mz->lruvec);
  5191. mz->usage_in_excess = 0;
  5192. mz->on_tree = false;
  5193. mz->memcg = memcg;
  5194. }
  5195. memcg->info.nodeinfo[node] = pn;
  5196. return 0;
  5197. }
  5198. static void free_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
  5199. {
  5200. kfree(memcg->info.nodeinfo[node]);
  5201. }
  5202. static struct mem_cgroup *mem_cgroup_alloc(void)
  5203. {
  5204. struct mem_cgroup *memcg;
  5205. size_t size = memcg_size();
  5206. /* Can be very big if nr_node_ids is very big */
  5207. if (size < PAGE_SIZE)
  5208. memcg = kzalloc(size, GFP_KERNEL);
  5209. else
  5210. memcg = vzalloc(size);
  5211. if (!memcg)
  5212. return NULL;
  5213. memcg->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
  5214. if (!memcg->stat)
  5215. goto out_free;
  5216. spin_lock_init(&memcg->pcp_counter_lock);
  5217. return memcg;
  5218. out_free:
  5219. if (size < PAGE_SIZE)
  5220. kfree(memcg);
  5221. else
  5222. vfree(memcg);
  5223. return NULL;
  5224. }
  5225. /*
  5226. * At destroying mem_cgroup, references from swap_cgroup can remain.
  5227. * (scanning all at force_empty is too costly...)
  5228. *
  5229. * Instead of clearing all references at force_empty, we remember
  5230. * the number of reference from swap_cgroup and free mem_cgroup when
  5231. * it goes down to 0.
  5232. *
  5233. * Removal of cgroup itself succeeds regardless of refs from swap.
  5234. */
  5235. static void __mem_cgroup_free(struct mem_cgroup *memcg)
  5236. {
  5237. int node;
  5238. size_t size = memcg_size();
  5239. mem_cgroup_remove_from_trees(memcg);
  5240. free_css_id(&mem_cgroup_subsys, &memcg->css);
  5241. for_each_node(node)
  5242. free_mem_cgroup_per_zone_info(memcg, node);
  5243. free_percpu(memcg->stat);
  5244. /*
  5245. * We need to make sure that (at least for now), the jump label
  5246. * destruction code runs outside of the cgroup lock. This is because
  5247. * get_online_cpus(), which is called from the static_branch update,
  5248. * can't be called inside the cgroup_lock. cpusets are the ones
  5249. * enforcing this dependency, so if they ever change, we might as well.
  5250. *
  5251. * schedule_work() will guarantee this happens. Be careful if you need
  5252. * to move this code around, and make sure it is outside
  5253. * the cgroup_lock.
  5254. */
  5255. disarm_static_keys(memcg);
  5256. if (size < PAGE_SIZE)
  5257. kfree(memcg);
  5258. else
  5259. vfree(memcg);
  5260. }
  5261. /*
  5262. * Helpers for freeing a kmalloc()ed/vzalloc()ed mem_cgroup by RCU,
  5263. * but in process context. The work_freeing structure is overlaid
  5264. * on the rcu_freeing structure, which itself is overlaid on memsw.
  5265. */
  5266. static void free_work(struct work_struct *work)
  5267. {
  5268. struct mem_cgroup *memcg;
  5269. memcg = container_of(work, struct mem_cgroup, work_freeing);
  5270. __mem_cgroup_free(memcg);
  5271. }
  5272. static void free_rcu(struct rcu_head *rcu_head)
  5273. {
  5274. struct mem_cgroup *memcg;
  5275. memcg = container_of(rcu_head, struct mem_cgroup, rcu_freeing);
  5276. INIT_WORK(&memcg->work_freeing, free_work);
  5277. schedule_work(&memcg->work_freeing);
  5278. }
  5279. static void mem_cgroup_get(struct mem_cgroup *memcg)
  5280. {
  5281. atomic_inc(&memcg->refcnt);
  5282. }
  5283. static void __mem_cgroup_put(struct mem_cgroup *memcg, int count)
  5284. {
  5285. if (atomic_sub_and_test(count, &memcg->refcnt)) {
  5286. struct mem_cgroup *parent = parent_mem_cgroup(memcg);
  5287. call_rcu(&memcg->rcu_freeing, free_rcu);
  5288. if (parent)
  5289. mem_cgroup_put(parent);
  5290. }
  5291. }
  5292. static void mem_cgroup_put(struct mem_cgroup *memcg)
  5293. {
  5294. __mem_cgroup_put(memcg, 1);
  5295. }
  5296. /*
  5297. * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
  5298. */
  5299. struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
  5300. {
  5301. if (!memcg->res.parent)
  5302. return NULL;
  5303. return mem_cgroup_from_res_counter(memcg->res.parent, res);
  5304. }
  5305. EXPORT_SYMBOL(parent_mem_cgroup);
  5306. static void __init mem_cgroup_soft_limit_tree_init(void)
  5307. {
  5308. struct mem_cgroup_tree_per_node *rtpn;
  5309. struct mem_cgroup_tree_per_zone *rtpz;
  5310. int tmp, node, zone;
  5311. for_each_node(node) {
  5312. tmp = node;
  5313. if (!node_state(node, N_NORMAL_MEMORY))
  5314. tmp = -1;
  5315. rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp);
  5316. BUG_ON(!rtpn);
  5317. soft_limit_tree.rb_tree_per_node[node] = rtpn;
  5318. for (zone = 0; zone < MAX_NR_ZONES; zone++) {
  5319. rtpz = &rtpn->rb_tree_per_zone[zone];
  5320. rtpz->rb_root = RB_ROOT;
  5321. spin_lock_init(&rtpz->lock);
  5322. }
  5323. }
  5324. }
  5325. static struct cgroup_subsys_state * __ref
  5326. mem_cgroup_css_alloc(struct cgroup *cont)
  5327. {
  5328. struct mem_cgroup *memcg;
  5329. long error = -ENOMEM;
  5330. int node;
  5331. memcg = mem_cgroup_alloc();
  5332. if (!memcg)
  5333. return ERR_PTR(error);
  5334. for_each_node(node)
  5335. if (alloc_mem_cgroup_per_zone_info(memcg, node))
  5336. goto free_out;
  5337. /* root ? */
  5338. if (cont->parent == NULL) {
  5339. root_mem_cgroup = memcg;
  5340. res_counter_init(&memcg->res, NULL);
  5341. res_counter_init(&memcg->memsw, NULL);
  5342. res_counter_init(&memcg->kmem, NULL);
  5343. }
  5344. memcg->last_scanned_node = MAX_NUMNODES;
  5345. INIT_LIST_HEAD(&memcg->oom_notify);
  5346. atomic_set(&memcg->refcnt, 1);
  5347. memcg->move_charge_at_immigrate = 0;
  5348. mutex_init(&memcg->thresholds_lock);
  5349. spin_lock_init(&memcg->move_lock);
  5350. return &memcg->css;
  5351. free_out:
  5352. __mem_cgroup_free(memcg);
  5353. return ERR_PTR(error);
  5354. }
  5355. static int
  5356. mem_cgroup_css_online(struct cgroup *cont)
  5357. {
  5358. struct mem_cgroup *memcg, *parent;
  5359. int error = 0;
  5360. if (!cont->parent)
  5361. return 0;
  5362. mutex_lock(&memcg_create_mutex);
  5363. memcg = mem_cgroup_from_cont(cont);
  5364. parent = mem_cgroup_from_cont(cont->parent);
  5365. memcg->use_hierarchy = parent->use_hierarchy;
  5366. memcg->oom_kill_disable = parent->oom_kill_disable;
  5367. memcg->swappiness = mem_cgroup_swappiness(parent);
  5368. if (parent->use_hierarchy) {
  5369. res_counter_init(&memcg->res, &parent->res);
  5370. res_counter_init(&memcg->memsw, &parent->memsw);
  5371. res_counter_init(&memcg->kmem, &parent->kmem);
  5372. /*
  5373. * We increment refcnt of the parent to ensure that we can
  5374. * safely access it on res_counter_charge/uncharge.
  5375. * This refcnt will be decremented when freeing this
  5376. * mem_cgroup(see mem_cgroup_put).
  5377. */
  5378. mem_cgroup_get(parent);
  5379. } else {
  5380. res_counter_init(&memcg->res, NULL);
  5381. res_counter_init(&memcg->memsw, NULL);
  5382. res_counter_init(&memcg->kmem, NULL);
  5383. /*
  5384. * Deeper hierachy with use_hierarchy == false doesn't make
  5385. * much sense so let cgroup subsystem know about this
  5386. * unfortunate state in our controller.
  5387. */
  5388. if (parent != root_mem_cgroup)
  5389. mem_cgroup_subsys.broken_hierarchy = true;
  5390. }
  5391. error = memcg_init_kmem(memcg, &mem_cgroup_subsys);
  5392. mutex_unlock(&memcg_create_mutex);
  5393. if (error) {
  5394. /*
  5395. * We call put now because our (and parent's) refcnts
  5396. * are already in place. mem_cgroup_put() will internally
  5397. * call __mem_cgroup_free, so return directly
  5398. */
  5399. mem_cgroup_put(memcg);
  5400. if (parent->use_hierarchy)
  5401. mem_cgroup_put(parent);
  5402. }
  5403. return error;
  5404. }
  5405. static void mem_cgroup_css_offline(struct cgroup *cont)
  5406. {
  5407. struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
  5408. mem_cgroup_reparent_charges(memcg);
  5409. mem_cgroup_destroy_all_caches(memcg);
  5410. }
  5411. static void mem_cgroup_css_free(struct cgroup *cont)
  5412. {
  5413. struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
  5414. kmem_cgroup_destroy(memcg);
  5415. mem_cgroup_put(memcg);
  5416. }
  5417. #ifdef CONFIG_MMU
  5418. /* Handlers for move charge at task migration. */
  5419. #define PRECHARGE_COUNT_AT_ONCE 256
  5420. static int mem_cgroup_do_precharge(unsigned long count)
  5421. {
  5422. int ret = 0;
  5423. int batch_count = PRECHARGE_COUNT_AT_ONCE;
  5424. struct mem_cgroup *memcg = mc.to;
  5425. if (mem_cgroup_is_root(memcg)) {
  5426. mc.precharge += count;
  5427. /* we don't need css_get for root */
  5428. return ret;
  5429. }
  5430. /* try to charge at once */
  5431. if (count > 1) {
  5432. struct res_counter *dummy;
  5433. /*
  5434. * "memcg" cannot be under rmdir() because we've already checked
  5435. * by cgroup_lock_live_cgroup() that it is not removed and we
  5436. * are still under the same cgroup_mutex. So we can postpone
  5437. * css_get().
  5438. */
  5439. if (res_counter_charge(&memcg->res, PAGE_SIZE * count, &dummy))
  5440. goto one_by_one;
  5441. if (do_swap_account && res_counter_charge(&memcg->memsw,
  5442. PAGE_SIZE * count, &dummy)) {
  5443. res_counter_uncharge(&memcg->res, PAGE_SIZE * count);
  5444. goto one_by_one;
  5445. }
  5446. mc.precharge += count;
  5447. return ret;
  5448. }
  5449. one_by_one:
  5450. /* fall back to one by one charge */
  5451. while (count--) {
  5452. if (signal_pending(current)) {
  5453. ret = -EINTR;
  5454. break;
  5455. }
  5456. if (!batch_count--) {
  5457. batch_count = PRECHARGE_COUNT_AT_ONCE;
  5458. cond_resched();
  5459. }
  5460. ret = __mem_cgroup_try_charge(NULL,
  5461. GFP_KERNEL, 1, &memcg, false);
  5462. if (ret)
  5463. /* mem_cgroup_clear_mc() will do uncharge later */
  5464. return ret;
  5465. mc.precharge++;
  5466. }
  5467. return ret;
  5468. }
  5469. /**
  5470. * get_mctgt_type - get target type of moving charge
  5471. * @vma: the vma the pte to be checked belongs
  5472. * @addr: the address corresponding to the pte to be checked
  5473. * @ptent: the pte to be checked
  5474. * @target: the pointer the target page or swap ent will be stored(can be NULL)
  5475. *
  5476. * Returns
  5477. * 0(MC_TARGET_NONE): if the pte is not a target for move charge.
  5478. * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
  5479. * move charge. if @target is not NULL, the page is stored in target->page
  5480. * with extra refcnt got(Callers should handle it).
  5481. * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
  5482. * target for charge migration. if @target is not NULL, the entry is stored
  5483. * in target->ent.
  5484. *
  5485. * Called with pte lock held.
  5486. */
  5487. union mc_target {
  5488. struct page *page;
  5489. swp_entry_t ent;
  5490. };
  5491. enum mc_target_type {
  5492. MC_TARGET_NONE = 0,
  5493. MC_TARGET_PAGE,
  5494. MC_TARGET_SWAP,
  5495. };
  5496. static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
  5497. unsigned long addr, pte_t ptent)
  5498. {
  5499. struct page *page = vm_normal_page(vma, addr, ptent);
  5500. if (!page || !page_mapped(page))
  5501. return NULL;
  5502. if (PageAnon(page)) {
  5503. /* we don't move shared anon */
  5504. if (!move_anon())
  5505. return NULL;
  5506. } else if (!move_file())
  5507. /* we ignore mapcount for file pages */
  5508. return NULL;
  5509. if (!get_page_unless_zero(page))
  5510. return NULL;
  5511. return page;
  5512. }
  5513. #ifdef CONFIG_SWAP
  5514. static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
  5515. unsigned long addr, pte_t ptent, swp_entry_t *entry)
  5516. {
  5517. struct page *page = NULL;
  5518. swp_entry_t ent = pte_to_swp_entry(ptent);
  5519. if (!move_anon() || non_swap_entry(ent))
  5520. return NULL;
  5521. /*
  5522. * Because lookup_swap_cache() updates some statistics counter,
  5523. * we call find_get_page() with swapper_space directly.
  5524. */
  5525. page = find_get_page(swap_address_space(ent), ent.val);
  5526. if (do_swap_account)
  5527. entry->val = ent.val;
  5528. return page;
  5529. }
  5530. #else
  5531. static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
  5532. unsigned long addr, pte_t ptent, swp_entry_t *entry)
  5533. {
  5534. return NULL;
  5535. }
  5536. #endif
  5537. static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
  5538. unsigned long addr, pte_t ptent, swp_entry_t *entry)
  5539. {
  5540. struct page *page = NULL;
  5541. struct address_space *mapping;
  5542. pgoff_t pgoff;
  5543. if (!vma->vm_file) /* anonymous vma */
  5544. return NULL;
  5545. if (!move_file())
  5546. return NULL;
  5547. mapping = vma->vm_file->f_mapping;
  5548. if (pte_none(ptent))
  5549. pgoff = linear_page_index(vma, addr);
  5550. else /* pte_file(ptent) is true */
  5551. pgoff = pte_to_pgoff(ptent);
  5552. /* page is moved even if it's not RSS of this task(page-faulted). */
  5553. page = find_get_page(mapping, pgoff);
  5554. #ifdef CONFIG_SWAP
  5555. /* shmem/tmpfs may report page out on swap: account for that too. */
  5556. if (radix_tree_exceptional_entry(page)) {
  5557. swp_entry_t swap = radix_to_swp_entry(page);
  5558. if (do_swap_account)
  5559. *entry = swap;
  5560. page = find_get_page(swap_address_space(swap), swap.val);
  5561. }
  5562. #endif
  5563. return page;
  5564. }
  5565. static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
  5566. unsigned long addr, pte_t ptent, union mc_target *target)
  5567. {
  5568. struct page *page = NULL;
  5569. struct page_cgroup *pc;
  5570. enum mc_target_type ret = MC_TARGET_NONE;
  5571. swp_entry_t ent = { .val = 0 };
  5572. if (pte_present(ptent))
  5573. page = mc_handle_present_pte(vma, addr, ptent);
  5574. else if (is_swap_pte(ptent))
  5575. page = mc_handle_swap_pte(vma, addr, ptent, &ent);
  5576. else if (pte_none(ptent) || pte_file(ptent))
  5577. page = mc_handle_file_pte(vma, addr, ptent, &ent);
  5578. if (!page && !ent.val)
  5579. return ret;
  5580. if (page) {
  5581. pc = lookup_page_cgroup(page);
  5582. /*
  5583. * Do only loose check w/o page_cgroup lock.
  5584. * mem_cgroup_move_account() checks the pc is valid or not under
  5585. * the lock.
  5586. */
  5587. if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
  5588. ret = MC_TARGET_PAGE;
  5589. if (target)
  5590. target->page = page;
  5591. }
  5592. if (!ret || !target)
  5593. put_page(page);
  5594. }
  5595. /* There is a swap entry and a page doesn't exist or isn't charged */
  5596. if (ent.val && !ret &&
  5597. css_id(&mc.from->css) == lookup_swap_cgroup_id(ent)) {
  5598. ret = MC_TARGET_SWAP;
  5599. if (target)
  5600. target->ent = ent;
  5601. }
  5602. return ret;
  5603. }
  5604. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  5605. /*
  5606. * We don't consider swapping or file mapped pages because THP does not
  5607. * support them for now.
  5608. * Caller should make sure that pmd_trans_huge(pmd) is true.
  5609. */
  5610. static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
  5611. unsigned long addr, pmd_t pmd, union mc_target *target)
  5612. {
  5613. struct page *page = NULL;
  5614. struct page_cgroup *pc;
  5615. enum mc_target_type ret = MC_TARGET_NONE;
  5616. page = pmd_page(pmd);
  5617. VM_BUG_ON(!page || !PageHead(page));
  5618. if (!move_anon())
  5619. return ret;
  5620. pc = lookup_page_cgroup(page);
  5621. if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
  5622. ret = MC_TARGET_PAGE;
  5623. if (target) {
  5624. get_page(page);
  5625. target->page = page;
  5626. }
  5627. }
  5628. return ret;
  5629. }
  5630. #else
  5631. static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
  5632. unsigned long addr, pmd_t pmd, union mc_target *target)
  5633. {
  5634. return MC_TARGET_NONE;
  5635. }
  5636. #endif
  5637. static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
  5638. unsigned long addr, unsigned long end,
  5639. struct mm_walk *walk)
  5640. {
  5641. struct vm_area_struct *vma = walk->private;
  5642. pte_t *pte;
  5643. spinlock_t *ptl;
  5644. if (pmd_trans_huge_lock(pmd, vma) == 1) {
  5645. if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
  5646. mc.precharge += HPAGE_PMD_NR;
  5647. spin_unlock(&vma->vm_mm->page_table_lock);
  5648. return 0;
  5649. }
  5650. if (pmd_trans_unstable(pmd))
  5651. return 0;
  5652. pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
  5653. for (; addr != end; pte++, addr += PAGE_SIZE)
  5654. if (get_mctgt_type(vma, addr, *pte, NULL))
  5655. mc.precharge++; /* increment precharge temporarily */
  5656. pte_unmap_unlock(pte - 1, ptl);
  5657. cond_resched();
  5658. return 0;
  5659. }
  5660. static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
  5661. {
  5662. unsigned long precharge;
  5663. struct vm_area_struct *vma;
  5664. down_read(&mm->mmap_sem);
  5665. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  5666. struct mm_walk mem_cgroup_count_precharge_walk = {
  5667. .pmd_entry = mem_cgroup_count_precharge_pte_range,
  5668. .mm = mm,
  5669. .private = vma,
  5670. };
  5671. if (is_vm_hugetlb_page(vma))
  5672. continue;
  5673. walk_page_range(vma->vm_start, vma->vm_end,
  5674. &mem_cgroup_count_precharge_walk);
  5675. }
  5676. up_read(&mm->mmap_sem);
  5677. precharge = mc.precharge;
  5678. mc.precharge = 0;
  5679. return precharge;
  5680. }
  5681. static int mem_cgroup_precharge_mc(struct mm_struct *mm)
  5682. {
  5683. unsigned long precharge = mem_cgroup_count_precharge(mm);
  5684. VM_BUG_ON(mc.moving_task);
  5685. mc.moving_task = current;
  5686. return mem_cgroup_do_precharge(precharge);
  5687. }
  5688. /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
  5689. static void __mem_cgroup_clear_mc(void)
  5690. {
  5691. struct mem_cgroup *from = mc.from;
  5692. struct mem_cgroup *to = mc.to;
  5693. /* we must uncharge all the leftover precharges from mc.to */
  5694. if (mc.precharge) {
  5695. __mem_cgroup_cancel_charge(mc.to, mc.precharge);
  5696. mc.precharge = 0;
  5697. }
  5698. /*
  5699. * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
  5700. * we must uncharge here.
  5701. */
  5702. if (mc.moved_charge) {
  5703. __mem_cgroup_cancel_charge(mc.from, mc.moved_charge);
  5704. mc.moved_charge = 0;
  5705. }
  5706. /* we must fixup refcnts and charges */
  5707. if (mc.moved_swap) {
  5708. /* uncharge swap account from the old cgroup */
  5709. if (!mem_cgroup_is_root(mc.from))
  5710. res_counter_uncharge(&mc.from->memsw,
  5711. PAGE_SIZE * mc.moved_swap);
  5712. __mem_cgroup_put(mc.from, mc.moved_swap);
  5713. if (!mem_cgroup_is_root(mc.to)) {
  5714. /*
  5715. * we charged both to->res and to->memsw, so we should
  5716. * uncharge to->res.
  5717. */
  5718. res_counter_uncharge(&mc.to->res,
  5719. PAGE_SIZE * mc.moved_swap);
  5720. }
  5721. /* we've already done mem_cgroup_get(mc.to) */
  5722. mc.moved_swap = 0;
  5723. }
  5724. memcg_oom_recover(from);
  5725. memcg_oom_recover(to);
  5726. wake_up_all(&mc.waitq);
  5727. }
  5728. static void mem_cgroup_clear_mc(void)
  5729. {
  5730. struct mem_cgroup *from = mc.from;
  5731. /*
  5732. * we must clear moving_task before waking up waiters at the end of
  5733. * task migration.
  5734. */
  5735. mc.moving_task = NULL;
  5736. __mem_cgroup_clear_mc();
  5737. spin_lock(&mc.lock);
  5738. mc.from = NULL;
  5739. mc.to = NULL;
  5740. spin_unlock(&mc.lock);
  5741. mem_cgroup_end_move(from);
  5742. }
  5743. static int mem_cgroup_can_attach(struct cgroup *cgroup,
  5744. struct cgroup_taskset *tset)
  5745. {
  5746. struct task_struct *p = cgroup_taskset_first(tset);
  5747. int ret = 0;
  5748. struct mem_cgroup *memcg = mem_cgroup_from_cont(cgroup);
  5749. unsigned long move_charge_at_immigrate;
  5750. /*
  5751. * We are now commited to this value whatever it is. Changes in this
  5752. * tunable will only affect upcoming migrations, not the current one.
  5753. * So we need to save it, and keep it going.
  5754. */
  5755. move_charge_at_immigrate = memcg->move_charge_at_immigrate;
  5756. if (move_charge_at_immigrate) {
  5757. struct mm_struct *mm;
  5758. struct mem_cgroup *from = mem_cgroup_from_task(p);
  5759. VM_BUG_ON(from == memcg);
  5760. mm = get_task_mm(p);
  5761. if (!mm)
  5762. return 0;
  5763. /* We move charges only when we move a owner of the mm */
  5764. if (mm->owner == p) {
  5765. VM_BUG_ON(mc.from);
  5766. VM_BUG_ON(mc.to);
  5767. VM_BUG_ON(mc.precharge);
  5768. VM_BUG_ON(mc.moved_charge);
  5769. VM_BUG_ON(mc.moved_swap);
  5770. mem_cgroup_start_move(from);
  5771. spin_lock(&mc.lock);
  5772. mc.from = from;
  5773. mc.to = memcg;
  5774. mc.immigrate_flags = move_charge_at_immigrate;
  5775. spin_unlock(&mc.lock);
  5776. /* We set mc.moving_task later */
  5777. ret = mem_cgroup_precharge_mc(mm);
  5778. if (ret)
  5779. mem_cgroup_clear_mc();
  5780. }
  5781. mmput(mm);
  5782. }
  5783. return ret;
  5784. }
  5785. static void mem_cgroup_cancel_attach(struct cgroup *cgroup,
  5786. struct cgroup_taskset *tset)
  5787. {
  5788. mem_cgroup_clear_mc();
  5789. }
  5790. static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
  5791. unsigned long addr, unsigned long end,
  5792. struct mm_walk *walk)
  5793. {
  5794. int ret = 0;
  5795. struct vm_area_struct *vma = walk->private;
  5796. pte_t *pte;
  5797. spinlock_t *ptl;
  5798. enum mc_target_type target_type;
  5799. union mc_target target;
  5800. struct page *page;
  5801. struct page_cgroup *pc;
  5802. /*
  5803. * We don't take compound_lock() here but no race with splitting thp
  5804. * happens because:
  5805. * - if pmd_trans_huge_lock() returns 1, the relevant thp is not
  5806. * under splitting, which means there's no concurrent thp split,
  5807. * - if another thread runs into split_huge_page() just after we
  5808. * entered this if-block, the thread must wait for page table lock
  5809. * to be unlocked in __split_huge_page_splitting(), where the main
  5810. * part of thp split is not executed yet.
  5811. */
  5812. if (pmd_trans_huge_lock(pmd, vma) == 1) {
  5813. if (mc.precharge < HPAGE_PMD_NR) {
  5814. spin_unlock(&vma->vm_mm->page_table_lock);
  5815. return 0;
  5816. }
  5817. target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
  5818. if (target_type == MC_TARGET_PAGE) {
  5819. page = target.page;
  5820. if (!isolate_lru_page(page)) {
  5821. pc = lookup_page_cgroup(page);
  5822. if (!mem_cgroup_move_account(page, HPAGE_PMD_NR,
  5823. pc, mc.from, mc.to)) {
  5824. mc.precharge -= HPAGE_PMD_NR;
  5825. mc.moved_charge += HPAGE_PMD_NR;
  5826. }
  5827. putback_lru_page(page);
  5828. }
  5829. put_page(page);
  5830. }
  5831. spin_unlock(&vma->vm_mm->page_table_lock);
  5832. return 0;
  5833. }
  5834. if (pmd_trans_unstable(pmd))
  5835. return 0;
  5836. retry:
  5837. pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
  5838. for (; addr != end; addr += PAGE_SIZE) {
  5839. pte_t ptent = *(pte++);
  5840. swp_entry_t ent;
  5841. if (!mc.precharge)
  5842. break;
  5843. switch (get_mctgt_type(vma, addr, ptent, &target)) {
  5844. case MC_TARGET_PAGE:
  5845. page = target.page;
  5846. if (isolate_lru_page(page))
  5847. goto put;
  5848. pc = lookup_page_cgroup(page);
  5849. if (!mem_cgroup_move_account(page, 1, pc,
  5850. mc.from, mc.to)) {
  5851. mc.precharge--;
  5852. /* we uncharge from mc.from later. */
  5853. mc.moved_charge++;
  5854. }
  5855. putback_lru_page(page);
  5856. put: /* get_mctgt_type() gets the page */
  5857. put_page(page);
  5858. break;
  5859. case MC_TARGET_SWAP:
  5860. ent = target.ent;
  5861. if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
  5862. mc.precharge--;
  5863. /* we fixup refcnts and charges later. */
  5864. mc.moved_swap++;
  5865. }
  5866. break;
  5867. default:
  5868. break;
  5869. }
  5870. }
  5871. pte_unmap_unlock(pte - 1, ptl);
  5872. cond_resched();
  5873. if (addr != end) {
  5874. /*
  5875. * We have consumed all precharges we got in can_attach().
  5876. * We try charge one by one, but don't do any additional
  5877. * charges to mc.to if we have failed in charge once in attach()
  5878. * phase.
  5879. */
  5880. ret = mem_cgroup_do_precharge(1);
  5881. if (!ret)
  5882. goto retry;
  5883. }
  5884. return ret;
  5885. }
  5886. static void mem_cgroup_move_charge(struct mm_struct *mm)
  5887. {
  5888. struct vm_area_struct *vma;
  5889. lru_add_drain_all();
  5890. retry:
  5891. if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
  5892. /*
  5893. * Someone who are holding the mmap_sem might be waiting in
  5894. * waitq. So we cancel all extra charges, wake up all waiters,
  5895. * and retry. Because we cancel precharges, we might not be able
  5896. * to move enough charges, but moving charge is a best-effort
  5897. * feature anyway, so it wouldn't be a big problem.
  5898. */
  5899. __mem_cgroup_clear_mc();
  5900. cond_resched();
  5901. goto retry;
  5902. }
  5903. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  5904. int ret;
  5905. struct mm_walk mem_cgroup_move_charge_walk = {
  5906. .pmd_entry = mem_cgroup_move_charge_pte_range,
  5907. .mm = mm,
  5908. .private = vma,
  5909. };
  5910. if (is_vm_hugetlb_page(vma))
  5911. continue;
  5912. ret = walk_page_range(vma->vm_start, vma->vm_end,
  5913. &mem_cgroup_move_charge_walk);
  5914. if (ret)
  5915. /*
  5916. * means we have consumed all precharges and failed in
  5917. * doing additional charge. Just abandon here.
  5918. */
  5919. break;
  5920. }
  5921. up_read(&mm->mmap_sem);
  5922. }
  5923. static void mem_cgroup_move_task(struct cgroup *cont,
  5924. struct cgroup_taskset *tset)
  5925. {
  5926. struct task_struct *p = cgroup_taskset_first(tset);
  5927. struct mm_struct *mm = get_task_mm(p);
  5928. if (mm) {
  5929. if (mc.to)
  5930. mem_cgroup_move_charge(mm);
  5931. mmput(mm);
  5932. }
  5933. if (mc.to)
  5934. mem_cgroup_clear_mc();
  5935. }
  5936. #else /* !CONFIG_MMU */
  5937. static int mem_cgroup_can_attach(struct cgroup *cgroup,
  5938. struct cgroup_taskset *tset)
  5939. {
  5940. return 0;
  5941. }
  5942. static void mem_cgroup_cancel_attach(struct cgroup *cgroup,
  5943. struct cgroup_taskset *tset)
  5944. {
  5945. }
  5946. static void mem_cgroup_move_task(struct cgroup *cont,
  5947. struct cgroup_taskset *tset)
  5948. {
  5949. }
  5950. #endif
  5951. struct cgroup_subsys mem_cgroup_subsys = {
  5952. .name = "memory",
  5953. .subsys_id = mem_cgroup_subsys_id,
  5954. .css_alloc = mem_cgroup_css_alloc,
  5955. .css_online = mem_cgroup_css_online,
  5956. .css_offline = mem_cgroup_css_offline,
  5957. .css_free = mem_cgroup_css_free,
  5958. .can_attach = mem_cgroup_can_attach,
  5959. .cancel_attach = mem_cgroup_cancel_attach,
  5960. .attach = mem_cgroup_move_task,
  5961. .base_cftypes = mem_cgroup_files,
  5962. .early_init = 0,
  5963. .use_id = 1,
  5964. };
  5965. #ifdef CONFIG_MEMCG_SWAP
  5966. static int __init enable_swap_account(char *s)
  5967. {
  5968. /* consider enabled if no parameter or 1 is given */
  5969. if (!strcmp(s, "1"))
  5970. really_do_swap_account = 1;
  5971. else if (!strcmp(s, "0"))
  5972. really_do_swap_account = 0;
  5973. return 1;
  5974. }
  5975. __setup("swapaccount=", enable_swap_account);
  5976. static void __init memsw_file_init(void)
  5977. {
  5978. WARN_ON(cgroup_add_cftypes(&mem_cgroup_subsys, memsw_cgroup_files));
  5979. }
  5980. static void __init enable_swap_cgroup(void)
  5981. {
  5982. if (!mem_cgroup_disabled() && really_do_swap_account) {
  5983. do_swap_account = 1;
  5984. memsw_file_init();
  5985. }
  5986. }
  5987. #else
  5988. static void __init enable_swap_cgroup(void)
  5989. {
  5990. }
  5991. #endif
  5992. /*
  5993. * subsys_initcall() for memory controller.
  5994. *
  5995. * Some parts like hotcpu_notifier() have to be initialized from this context
  5996. * because of lock dependencies (cgroup_lock -> cpu hotplug) but basically
  5997. * everything that doesn't depend on a specific mem_cgroup structure should
  5998. * be initialized from here.
  5999. */
  6000. static int __init mem_cgroup_init(void)
  6001. {
  6002. hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
  6003. enable_swap_cgroup();
  6004. mem_cgroup_soft_limit_tree_init();
  6005. memcg_stock_init();
  6006. return 0;
  6007. }
  6008. subsys_initcall(mem_cgroup_init);