memcontrol.c 186 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605660666076608660966106611661266136614661566166617661866196620662166226623662466256626662766286629663066316632663366346635663666376638663966406641664266436644664566466647664866496650665166526653665466556656665766586659666066616662666366646665666666676668666966706671667266736674667566766677667866796680668166826683668466856686668766886689669066916692669366946695669666976698669967006701670267036704670567066707670867096710671167126713671467156716671767186719672067216722672367246725672667276728672967306731673267336734673567366737673867396740674167426743674467456746674767486749675067516752675367546755675667576758675967606761676267636764676567666767676867696770677167726773677467756776677767786779678067816782678367846785678667876788678967906791679267936794679567966797679867996800680168026803680468056806680768086809681068116812681368146815681668176818681968206821682268236824682568266827682868296830683168326833683468356836683768386839684068416842684368446845684668476848684968506851685268536854685568566857685868596860686168626863686468656866686768686869687068716872687368746875687668776878687968806881688268836884688568866887688868896890689168926893689468956896689768986899690069016902690369046905690669076908690969106911691269136914691569166917691869196920692169226923692469256926692769286929693069316932693369346935693669376938693969406941694269436944694569466947694869496950695169526953695469556956695769586959696069616962696369646965696669676968696969706971697269736974697569766977697869796980698169826983698469856986698769886989699069916992699369946995699669976998699970007001700270037004700570067007700870097010701170127013701470157016701770187019702070217022702370247025
  1. /* memcontrol.c - Memory Controller
  2. *
  3. * Copyright IBM Corporation, 2007
  4. * Author Balbir Singh <balbir@linux.vnet.ibm.com>
  5. *
  6. * Copyright 2007 OpenVZ SWsoft Inc
  7. * Author: Pavel Emelianov <xemul@openvz.org>
  8. *
  9. * Memory thresholds
  10. * Copyright (C) 2009 Nokia Corporation
  11. * Author: Kirill A. Shutemov
  12. *
  13. * Kernel Memory Controller
  14. * Copyright (C) 2012 Parallels Inc. and Google Inc.
  15. * Authors: Glauber Costa and Suleiman Souhlal
  16. *
  17. * This program is free software; you can redistribute it and/or modify
  18. * it under the terms of the GNU General Public License as published by
  19. * the Free Software Foundation; either version 2 of the License, or
  20. * (at your option) any later version.
  21. *
  22. * This program is distributed in the hope that it will be useful,
  23. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  24. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  25. * GNU General Public License for more details.
  26. */
  27. #include <linux/res_counter.h>
  28. #include <linux/memcontrol.h>
  29. #include <linux/cgroup.h>
  30. #include <linux/mm.h>
  31. #include <linux/hugetlb.h>
  32. #include <linux/pagemap.h>
  33. #include <linux/smp.h>
  34. #include <linux/page-flags.h>
  35. #include <linux/backing-dev.h>
  36. #include <linux/bit_spinlock.h>
  37. #include <linux/rcupdate.h>
  38. #include <linux/limits.h>
  39. #include <linux/export.h>
  40. #include <linux/mutex.h>
  41. #include <linux/rbtree.h>
  42. #include <linux/slab.h>
  43. #include <linux/swap.h>
  44. #include <linux/swapops.h>
  45. #include <linux/spinlock.h>
  46. #include <linux/eventfd.h>
  47. #include <linux/sort.h>
  48. #include <linux/fs.h>
  49. #include <linux/seq_file.h>
  50. #include <linux/vmalloc.h>
  51. #include <linux/vmpressure.h>
  52. #include <linux/mm_inline.h>
  53. #include <linux/page_cgroup.h>
  54. #include <linux/cpu.h>
  55. #include <linux/oom.h>
  56. #include <linux/lockdep.h>
  57. #include "internal.h"
  58. #include <net/sock.h>
  59. #include <net/ip.h>
  60. #include <net/tcp_memcontrol.h>
  61. #include "slab.h"
  62. #include <asm/uaccess.h>
  63. #include <trace/events/vmscan.h>
  64. struct cgroup_subsys mem_cgroup_subsys __read_mostly;
  65. EXPORT_SYMBOL(mem_cgroup_subsys);
  66. #define MEM_CGROUP_RECLAIM_RETRIES 5
  67. static struct mem_cgroup *root_mem_cgroup __read_mostly;
  68. #ifdef CONFIG_MEMCG_SWAP
  69. /* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */
  70. int do_swap_account __read_mostly;
  71. /* for remember boot option*/
  72. #ifdef CONFIG_MEMCG_SWAP_ENABLED
  73. static int really_do_swap_account __initdata = 1;
  74. #else
  75. static int really_do_swap_account __initdata = 0;
  76. #endif
  77. #else
  78. #define do_swap_account 0
  79. #endif
  80. static const char * const mem_cgroup_stat_names[] = {
  81. "cache",
  82. "rss",
  83. "rss_huge",
  84. "mapped_file",
  85. "writeback",
  86. "swap",
  87. };
  88. enum mem_cgroup_events_index {
  89. MEM_CGROUP_EVENTS_PGPGIN, /* # of pages paged in */
  90. MEM_CGROUP_EVENTS_PGPGOUT, /* # of pages paged out */
  91. MEM_CGROUP_EVENTS_PGFAULT, /* # of page-faults */
  92. MEM_CGROUP_EVENTS_PGMAJFAULT, /* # of major page-faults */
  93. MEM_CGROUP_EVENTS_NSTATS,
  94. };
  95. static const char * const mem_cgroup_events_names[] = {
  96. "pgpgin",
  97. "pgpgout",
  98. "pgfault",
  99. "pgmajfault",
  100. };
  101. static const char * const mem_cgroup_lru_names[] = {
  102. "inactive_anon",
  103. "active_anon",
  104. "inactive_file",
  105. "active_file",
  106. "unevictable",
  107. };
  108. /*
  109. * Per memcg event counter is incremented at every pagein/pageout. With THP,
  110. * it will be incremated by the number of pages. This counter is used for
  111. * for trigger some periodic events. This is straightforward and better
  112. * than using jiffies etc. to handle periodic memcg event.
  113. */
  114. enum mem_cgroup_events_target {
  115. MEM_CGROUP_TARGET_THRESH,
  116. MEM_CGROUP_TARGET_SOFTLIMIT,
  117. MEM_CGROUP_TARGET_NUMAINFO,
  118. MEM_CGROUP_NTARGETS,
  119. };
  120. #define THRESHOLDS_EVENTS_TARGET 128
  121. #define SOFTLIMIT_EVENTS_TARGET 1024
  122. #define NUMAINFO_EVENTS_TARGET 1024
  123. struct mem_cgroup_stat_cpu {
  124. long count[MEM_CGROUP_STAT_NSTATS];
  125. unsigned long events[MEM_CGROUP_EVENTS_NSTATS];
  126. unsigned long nr_page_events;
  127. unsigned long targets[MEM_CGROUP_NTARGETS];
  128. };
  129. struct mem_cgroup_reclaim_iter {
  130. /*
  131. * last scanned hierarchy member. Valid only if last_dead_count
  132. * matches memcg->dead_count of the hierarchy root group.
  133. */
  134. struct mem_cgroup *last_visited;
  135. unsigned long last_dead_count;
  136. /* scan generation, increased every round-trip */
  137. unsigned int generation;
  138. };
  139. /*
  140. * per-zone information in memory controller.
  141. */
  142. struct mem_cgroup_per_zone {
  143. struct lruvec lruvec;
  144. unsigned long lru_size[NR_LRU_LISTS];
  145. struct mem_cgroup_reclaim_iter reclaim_iter[DEF_PRIORITY + 1];
  146. struct rb_node tree_node; /* RB tree node */
  147. unsigned long long usage_in_excess;/* Set to the value by which */
  148. /* the soft limit is exceeded*/
  149. bool on_tree;
  150. struct mem_cgroup *memcg; /* Back pointer, we cannot */
  151. /* use container_of */
  152. };
  153. struct mem_cgroup_per_node {
  154. struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
  155. };
  156. /*
  157. * Cgroups above their limits are maintained in a RB-Tree, independent of
  158. * their hierarchy representation
  159. */
  160. struct mem_cgroup_tree_per_zone {
  161. struct rb_root rb_root;
  162. spinlock_t lock;
  163. };
  164. struct mem_cgroup_tree_per_node {
  165. struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
  166. };
  167. struct mem_cgroup_tree {
  168. struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
  169. };
  170. static struct mem_cgroup_tree soft_limit_tree __read_mostly;
  171. struct mem_cgroup_threshold {
  172. struct eventfd_ctx *eventfd;
  173. u64 threshold;
  174. };
  175. /* For threshold */
  176. struct mem_cgroup_threshold_ary {
  177. /* An array index points to threshold just below or equal to usage. */
  178. int current_threshold;
  179. /* Size of entries[] */
  180. unsigned int size;
  181. /* Array of thresholds */
  182. struct mem_cgroup_threshold entries[0];
  183. };
  184. struct mem_cgroup_thresholds {
  185. /* Primary thresholds array */
  186. struct mem_cgroup_threshold_ary *primary;
  187. /*
  188. * Spare threshold array.
  189. * This is needed to make mem_cgroup_unregister_event() "never fail".
  190. * It must be able to store at least primary->size - 1 entries.
  191. */
  192. struct mem_cgroup_threshold_ary *spare;
  193. };
  194. /* for OOM */
  195. struct mem_cgroup_eventfd_list {
  196. struct list_head list;
  197. struct eventfd_ctx *eventfd;
  198. };
  199. static void mem_cgroup_threshold(struct mem_cgroup *memcg);
  200. static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
  201. /*
  202. * The memory controller data structure. The memory controller controls both
  203. * page cache and RSS per cgroup. We would eventually like to provide
  204. * statistics based on the statistics developed by Rik Van Riel for clock-pro,
  205. * to help the administrator determine what knobs to tune.
  206. *
  207. * TODO: Add a water mark for the memory controller. Reclaim will begin when
  208. * we hit the water mark. May be even add a low water mark, such that
  209. * no reclaim occurs from a cgroup at it's low water mark, this is
  210. * a feature that will be implemented much later in the future.
  211. */
  212. struct mem_cgroup {
  213. struct cgroup_subsys_state css;
  214. /*
  215. * the counter to account for memory usage
  216. */
  217. struct res_counter res;
  218. /* vmpressure notifications */
  219. struct vmpressure vmpressure;
  220. /*
  221. * the counter to account for mem+swap usage.
  222. */
  223. struct res_counter memsw;
  224. /*
  225. * the counter to account for kernel memory usage.
  226. */
  227. struct res_counter kmem;
  228. /*
  229. * Should the accounting and control be hierarchical, per subtree?
  230. */
  231. bool use_hierarchy;
  232. unsigned long kmem_account_flags; /* See KMEM_ACCOUNTED_*, below */
  233. bool oom_lock;
  234. atomic_t under_oom;
  235. atomic_t oom_wakeups;
  236. int swappiness;
  237. /* OOM-Killer disable */
  238. int oom_kill_disable;
  239. /* set when res.limit == memsw.limit */
  240. bool memsw_is_minimum;
  241. /* protect arrays of thresholds */
  242. struct mutex thresholds_lock;
  243. /* thresholds for memory usage. RCU-protected */
  244. struct mem_cgroup_thresholds thresholds;
  245. /* thresholds for mem+swap usage. RCU-protected */
  246. struct mem_cgroup_thresholds memsw_thresholds;
  247. /* For oom notifier event fd */
  248. struct list_head oom_notify;
  249. /*
  250. * Should we move charges of a task when a task is moved into this
  251. * mem_cgroup ? And what type of charges should we move ?
  252. */
  253. unsigned long move_charge_at_immigrate;
  254. /*
  255. * set > 0 if pages under this cgroup are moving to other cgroup.
  256. */
  257. atomic_t moving_account;
  258. /* taken only while moving_account > 0 */
  259. spinlock_t move_lock;
  260. /*
  261. * percpu counter.
  262. */
  263. struct mem_cgroup_stat_cpu __percpu *stat;
  264. /*
  265. * used when a cpu is offlined or other synchronizations
  266. * See mem_cgroup_read_stat().
  267. */
  268. struct mem_cgroup_stat_cpu nocpu_base;
  269. spinlock_t pcp_counter_lock;
  270. atomic_t dead_count;
  271. #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_INET)
  272. struct cg_proto tcp_mem;
  273. #endif
  274. #if defined(CONFIG_MEMCG_KMEM)
  275. /* analogous to slab_common's slab_caches list. per-memcg */
  276. struct list_head memcg_slab_caches;
  277. /* Not a spinlock, we can take a lot of time walking the list */
  278. struct mutex slab_caches_mutex;
  279. /* Index in the kmem_cache->memcg_params->memcg_caches array */
  280. int kmemcg_id;
  281. #endif
  282. int last_scanned_node;
  283. #if MAX_NUMNODES > 1
  284. nodemask_t scan_nodes;
  285. atomic_t numainfo_events;
  286. atomic_t numainfo_updating;
  287. #endif
  288. struct mem_cgroup_per_node *nodeinfo[0];
  289. /* WARNING: nodeinfo must be the last member here */
  290. };
  291. static size_t memcg_size(void)
  292. {
  293. return sizeof(struct mem_cgroup) +
  294. nr_node_ids * sizeof(struct mem_cgroup_per_node);
  295. }
  296. /* internal only representation about the status of kmem accounting. */
  297. enum {
  298. KMEM_ACCOUNTED_ACTIVE = 0, /* accounted by this cgroup itself */
  299. KMEM_ACCOUNTED_ACTIVATED, /* static key enabled. */
  300. KMEM_ACCOUNTED_DEAD, /* dead memcg with pending kmem charges */
  301. };
  302. /* We account when limit is on, but only after call sites are patched */
  303. #define KMEM_ACCOUNTED_MASK \
  304. ((1 << KMEM_ACCOUNTED_ACTIVE) | (1 << KMEM_ACCOUNTED_ACTIVATED))
  305. #ifdef CONFIG_MEMCG_KMEM
  306. static inline void memcg_kmem_set_active(struct mem_cgroup *memcg)
  307. {
  308. set_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags);
  309. }
  310. static bool memcg_kmem_is_active(struct mem_cgroup *memcg)
  311. {
  312. return test_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags);
  313. }
  314. static void memcg_kmem_set_activated(struct mem_cgroup *memcg)
  315. {
  316. set_bit(KMEM_ACCOUNTED_ACTIVATED, &memcg->kmem_account_flags);
  317. }
  318. static void memcg_kmem_clear_activated(struct mem_cgroup *memcg)
  319. {
  320. clear_bit(KMEM_ACCOUNTED_ACTIVATED, &memcg->kmem_account_flags);
  321. }
  322. static void memcg_kmem_mark_dead(struct mem_cgroup *memcg)
  323. {
  324. /*
  325. * Our caller must use css_get() first, because memcg_uncharge_kmem()
  326. * will call css_put() if it sees the memcg is dead.
  327. */
  328. smp_wmb();
  329. if (test_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags))
  330. set_bit(KMEM_ACCOUNTED_DEAD, &memcg->kmem_account_flags);
  331. }
  332. static bool memcg_kmem_test_and_clear_dead(struct mem_cgroup *memcg)
  333. {
  334. return test_and_clear_bit(KMEM_ACCOUNTED_DEAD,
  335. &memcg->kmem_account_flags);
  336. }
  337. #endif
  338. /* Stuffs for move charges at task migration. */
  339. /*
  340. * Types of charges to be moved. "move_charge_at_immitgrate" and
  341. * "immigrate_flags" are treated as a left-shifted bitmap of these types.
  342. */
  343. enum move_type {
  344. MOVE_CHARGE_TYPE_ANON, /* private anonymous page and swap of it */
  345. MOVE_CHARGE_TYPE_FILE, /* file page(including tmpfs) and swap of it */
  346. NR_MOVE_TYPE,
  347. };
  348. /* "mc" and its members are protected by cgroup_mutex */
  349. static struct move_charge_struct {
  350. spinlock_t lock; /* for from, to */
  351. struct mem_cgroup *from;
  352. struct mem_cgroup *to;
  353. unsigned long immigrate_flags;
  354. unsigned long precharge;
  355. unsigned long moved_charge;
  356. unsigned long moved_swap;
  357. struct task_struct *moving_task; /* a task moving charges */
  358. wait_queue_head_t waitq; /* a waitq for other context */
  359. } mc = {
  360. .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
  361. .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
  362. };
  363. static bool move_anon(void)
  364. {
  365. return test_bit(MOVE_CHARGE_TYPE_ANON, &mc.immigrate_flags);
  366. }
  367. static bool move_file(void)
  368. {
  369. return test_bit(MOVE_CHARGE_TYPE_FILE, &mc.immigrate_flags);
  370. }
  371. /*
  372. * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
  373. * limit reclaim to prevent infinite loops, if they ever occur.
  374. */
  375. #define MEM_CGROUP_MAX_RECLAIM_LOOPS 100
  376. #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
  377. enum charge_type {
  378. MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
  379. MEM_CGROUP_CHARGE_TYPE_ANON,
  380. MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
  381. MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */
  382. NR_CHARGE_TYPE,
  383. };
  384. /* for encoding cft->private value on file */
  385. enum res_type {
  386. _MEM,
  387. _MEMSWAP,
  388. _OOM_TYPE,
  389. _KMEM,
  390. };
  391. #define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))
  392. #define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff)
  393. #define MEMFILE_ATTR(val) ((val) & 0xffff)
  394. /* Used for OOM nofiier */
  395. #define OOM_CONTROL (0)
  396. /*
  397. * Reclaim flags for mem_cgroup_hierarchical_reclaim
  398. */
  399. #define MEM_CGROUP_RECLAIM_NOSWAP_BIT 0x0
  400. #define MEM_CGROUP_RECLAIM_NOSWAP (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
  401. #define MEM_CGROUP_RECLAIM_SHRINK_BIT 0x1
  402. #define MEM_CGROUP_RECLAIM_SHRINK (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
  403. /*
  404. * The memcg_create_mutex will be held whenever a new cgroup is created.
  405. * As a consequence, any change that needs to protect against new child cgroups
  406. * appearing has to hold it as well.
  407. */
  408. static DEFINE_MUTEX(memcg_create_mutex);
  409. struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *s)
  410. {
  411. return s ? container_of(s, struct mem_cgroup, css) : NULL;
  412. }
  413. /* Some nice accessors for the vmpressure. */
  414. struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
  415. {
  416. if (!memcg)
  417. memcg = root_mem_cgroup;
  418. return &memcg->vmpressure;
  419. }
  420. struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
  421. {
  422. return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
  423. }
  424. struct vmpressure *css_to_vmpressure(struct cgroup_subsys_state *css)
  425. {
  426. return &mem_cgroup_from_css(css)->vmpressure;
  427. }
  428. static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
  429. {
  430. return (memcg == root_mem_cgroup);
  431. }
  432. /*
  433. * We restrict the id in the range of [1, 65535], so it can fit into
  434. * an unsigned short.
  435. */
  436. #define MEM_CGROUP_ID_MAX USHRT_MAX
  437. static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
  438. {
  439. /*
  440. * The ID of the root cgroup is 0, but memcg treat 0 as an
  441. * invalid ID, so we return (cgroup_id + 1).
  442. */
  443. return memcg->css.cgroup->id + 1;
  444. }
  445. static inline struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
  446. {
  447. struct cgroup_subsys_state *css;
  448. css = css_from_id(id - 1, &mem_cgroup_subsys);
  449. return mem_cgroup_from_css(css);
  450. }
  451. /* Writing them here to avoid exposing memcg's inner layout */
  452. #if defined(CONFIG_INET) && defined(CONFIG_MEMCG_KMEM)
  453. void sock_update_memcg(struct sock *sk)
  454. {
  455. if (mem_cgroup_sockets_enabled) {
  456. struct mem_cgroup *memcg;
  457. struct cg_proto *cg_proto;
  458. BUG_ON(!sk->sk_prot->proto_cgroup);
  459. /* Socket cloning can throw us here with sk_cgrp already
  460. * filled. It won't however, necessarily happen from
  461. * process context. So the test for root memcg given
  462. * the current task's memcg won't help us in this case.
  463. *
  464. * Respecting the original socket's memcg is a better
  465. * decision in this case.
  466. */
  467. if (sk->sk_cgrp) {
  468. BUG_ON(mem_cgroup_is_root(sk->sk_cgrp->memcg));
  469. css_get(&sk->sk_cgrp->memcg->css);
  470. return;
  471. }
  472. rcu_read_lock();
  473. memcg = mem_cgroup_from_task(current);
  474. cg_proto = sk->sk_prot->proto_cgroup(memcg);
  475. if (!mem_cgroup_is_root(memcg) &&
  476. memcg_proto_active(cg_proto) && css_tryget(&memcg->css)) {
  477. sk->sk_cgrp = cg_proto;
  478. }
  479. rcu_read_unlock();
  480. }
  481. }
  482. EXPORT_SYMBOL(sock_update_memcg);
  483. void sock_release_memcg(struct sock *sk)
  484. {
  485. if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
  486. struct mem_cgroup *memcg;
  487. WARN_ON(!sk->sk_cgrp->memcg);
  488. memcg = sk->sk_cgrp->memcg;
  489. css_put(&sk->sk_cgrp->memcg->css);
  490. }
  491. }
  492. struct cg_proto *tcp_proto_cgroup(struct mem_cgroup *memcg)
  493. {
  494. if (!memcg || mem_cgroup_is_root(memcg))
  495. return NULL;
  496. return &memcg->tcp_mem;
  497. }
  498. EXPORT_SYMBOL(tcp_proto_cgroup);
  499. static void disarm_sock_keys(struct mem_cgroup *memcg)
  500. {
  501. if (!memcg_proto_activated(&memcg->tcp_mem))
  502. return;
  503. static_key_slow_dec(&memcg_socket_limit_enabled);
  504. }
  505. #else
  506. static void disarm_sock_keys(struct mem_cgroup *memcg)
  507. {
  508. }
  509. #endif
  510. #ifdef CONFIG_MEMCG_KMEM
  511. /*
  512. * This will be the memcg's index in each cache's ->memcg_params->memcg_caches.
  513. * The main reason for not using cgroup id for this:
  514. * this works better in sparse environments, where we have a lot of memcgs,
  515. * but only a few kmem-limited. Or also, if we have, for instance, 200
  516. * memcgs, and none but the 200th is kmem-limited, we'd have to have a
  517. * 200 entry array for that.
  518. *
  519. * The current size of the caches array is stored in
  520. * memcg_limited_groups_array_size. It will double each time we have to
  521. * increase it.
  522. */
  523. static DEFINE_IDA(kmem_limited_groups);
  524. int memcg_limited_groups_array_size;
  525. /*
  526. * MIN_SIZE is different than 1, because we would like to avoid going through
  527. * the alloc/free process all the time. In a small machine, 4 kmem-limited
  528. * cgroups is a reasonable guess. In the future, it could be a parameter or
  529. * tunable, but that is strictly not necessary.
  530. *
  531. * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get
  532. * this constant directly from cgroup, but it is understandable that this is
  533. * better kept as an internal representation in cgroup.c. In any case, the
  534. * cgrp_id space is not getting any smaller, and we don't have to necessarily
  535. * increase ours as well if it increases.
  536. */
  537. #define MEMCG_CACHES_MIN_SIZE 4
  538. #define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX
  539. /*
  540. * A lot of the calls to the cache allocation functions are expected to be
  541. * inlined by the compiler. Since the calls to memcg_kmem_get_cache are
  542. * conditional to this static branch, we'll have to allow modules that does
  543. * kmem_cache_alloc and the such to see this symbol as well
  544. */
  545. struct static_key memcg_kmem_enabled_key;
  546. EXPORT_SYMBOL(memcg_kmem_enabled_key);
  547. static void disarm_kmem_keys(struct mem_cgroup *memcg)
  548. {
  549. if (memcg_kmem_is_active(memcg)) {
  550. static_key_slow_dec(&memcg_kmem_enabled_key);
  551. ida_simple_remove(&kmem_limited_groups, memcg->kmemcg_id);
  552. }
  553. /*
  554. * This check can't live in kmem destruction function,
  555. * since the charges will outlive the cgroup
  556. */
  557. WARN_ON(res_counter_read_u64(&memcg->kmem, RES_USAGE) != 0);
  558. }
  559. #else
  560. static void disarm_kmem_keys(struct mem_cgroup *memcg)
  561. {
  562. }
  563. #endif /* CONFIG_MEMCG_KMEM */
  564. static void disarm_static_keys(struct mem_cgroup *memcg)
  565. {
  566. disarm_sock_keys(memcg);
  567. disarm_kmem_keys(memcg);
  568. }
  569. static void drain_all_stock_async(struct mem_cgroup *memcg);
  570. static struct mem_cgroup_per_zone *
  571. mem_cgroup_zoneinfo(struct mem_cgroup *memcg, int nid, int zid)
  572. {
  573. VM_BUG_ON((unsigned)nid >= nr_node_ids);
  574. return &memcg->nodeinfo[nid]->zoneinfo[zid];
  575. }
  576. struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg)
  577. {
  578. return &memcg->css;
  579. }
  580. static struct mem_cgroup_per_zone *
  581. page_cgroup_zoneinfo(struct mem_cgroup *memcg, struct page *page)
  582. {
  583. int nid = page_to_nid(page);
  584. int zid = page_zonenum(page);
  585. return mem_cgroup_zoneinfo(memcg, nid, zid);
  586. }
  587. static struct mem_cgroup_tree_per_zone *
  588. soft_limit_tree_node_zone(int nid, int zid)
  589. {
  590. return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
  591. }
  592. static struct mem_cgroup_tree_per_zone *
  593. soft_limit_tree_from_page(struct page *page)
  594. {
  595. int nid = page_to_nid(page);
  596. int zid = page_zonenum(page);
  597. return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
  598. }
  599. static void
  600. __mem_cgroup_insert_exceeded(struct mem_cgroup *memcg,
  601. struct mem_cgroup_per_zone *mz,
  602. struct mem_cgroup_tree_per_zone *mctz,
  603. unsigned long long new_usage_in_excess)
  604. {
  605. struct rb_node **p = &mctz->rb_root.rb_node;
  606. struct rb_node *parent = NULL;
  607. struct mem_cgroup_per_zone *mz_node;
  608. if (mz->on_tree)
  609. return;
  610. mz->usage_in_excess = new_usage_in_excess;
  611. if (!mz->usage_in_excess)
  612. return;
  613. while (*p) {
  614. parent = *p;
  615. mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
  616. tree_node);
  617. if (mz->usage_in_excess < mz_node->usage_in_excess)
  618. p = &(*p)->rb_left;
  619. /*
  620. * We can't avoid mem cgroups that are over their soft
  621. * limit by the same amount
  622. */
  623. else if (mz->usage_in_excess >= mz_node->usage_in_excess)
  624. p = &(*p)->rb_right;
  625. }
  626. rb_link_node(&mz->tree_node, parent, p);
  627. rb_insert_color(&mz->tree_node, &mctz->rb_root);
  628. mz->on_tree = true;
  629. }
  630. static void
  631. __mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
  632. struct mem_cgroup_per_zone *mz,
  633. struct mem_cgroup_tree_per_zone *mctz)
  634. {
  635. if (!mz->on_tree)
  636. return;
  637. rb_erase(&mz->tree_node, &mctz->rb_root);
  638. mz->on_tree = false;
  639. }
  640. static void
  641. mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
  642. struct mem_cgroup_per_zone *mz,
  643. struct mem_cgroup_tree_per_zone *mctz)
  644. {
  645. spin_lock(&mctz->lock);
  646. __mem_cgroup_remove_exceeded(memcg, mz, mctz);
  647. spin_unlock(&mctz->lock);
  648. }
  649. static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
  650. {
  651. unsigned long long excess;
  652. struct mem_cgroup_per_zone *mz;
  653. struct mem_cgroup_tree_per_zone *mctz;
  654. int nid = page_to_nid(page);
  655. int zid = page_zonenum(page);
  656. mctz = soft_limit_tree_from_page(page);
  657. /*
  658. * Necessary to update all ancestors when hierarchy is used.
  659. * because their event counter is not touched.
  660. */
  661. for (; memcg; memcg = parent_mem_cgroup(memcg)) {
  662. mz = mem_cgroup_zoneinfo(memcg, nid, zid);
  663. excess = res_counter_soft_limit_excess(&memcg->res);
  664. /*
  665. * We have to update the tree if mz is on RB-tree or
  666. * mem is over its softlimit.
  667. */
  668. if (excess || mz->on_tree) {
  669. spin_lock(&mctz->lock);
  670. /* if on-tree, remove it */
  671. if (mz->on_tree)
  672. __mem_cgroup_remove_exceeded(memcg, mz, mctz);
  673. /*
  674. * Insert again. mz->usage_in_excess will be updated.
  675. * If excess is 0, no tree ops.
  676. */
  677. __mem_cgroup_insert_exceeded(memcg, mz, mctz, excess);
  678. spin_unlock(&mctz->lock);
  679. }
  680. }
  681. }
  682. static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
  683. {
  684. int node, zone;
  685. struct mem_cgroup_per_zone *mz;
  686. struct mem_cgroup_tree_per_zone *mctz;
  687. for_each_node(node) {
  688. for (zone = 0; zone < MAX_NR_ZONES; zone++) {
  689. mz = mem_cgroup_zoneinfo(memcg, node, zone);
  690. mctz = soft_limit_tree_node_zone(node, zone);
  691. mem_cgroup_remove_exceeded(memcg, mz, mctz);
  692. }
  693. }
  694. }
  695. static struct mem_cgroup_per_zone *
  696. __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
  697. {
  698. struct rb_node *rightmost = NULL;
  699. struct mem_cgroup_per_zone *mz;
  700. retry:
  701. mz = NULL;
  702. rightmost = rb_last(&mctz->rb_root);
  703. if (!rightmost)
  704. goto done; /* Nothing to reclaim from */
  705. mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
  706. /*
  707. * Remove the node now but someone else can add it back,
  708. * we will to add it back at the end of reclaim to its correct
  709. * position in the tree.
  710. */
  711. __mem_cgroup_remove_exceeded(mz->memcg, mz, mctz);
  712. if (!res_counter_soft_limit_excess(&mz->memcg->res) ||
  713. !css_tryget(&mz->memcg->css))
  714. goto retry;
  715. done:
  716. return mz;
  717. }
  718. static struct mem_cgroup_per_zone *
  719. mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
  720. {
  721. struct mem_cgroup_per_zone *mz;
  722. spin_lock(&mctz->lock);
  723. mz = __mem_cgroup_largest_soft_limit_node(mctz);
  724. spin_unlock(&mctz->lock);
  725. return mz;
  726. }
  727. /*
  728. * Implementation Note: reading percpu statistics for memcg.
  729. *
  730. * Both of vmstat[] and percpu_counter has threshold and do periodic
  731. * synchronization to implement "quick" read. There are trade-off between
  732. * reading cost and precision of value. Then, we may have a chance to implement
  733. * a periodic synchronizion of counter in memcg's counter.
  734. *
  735. * But this _read() function is used for user interface now. The user accounts
  736. * memory usage by memory cgroup and he _always_ requires exact value because
  737. * he accounts memory. Even if we provide quick-and-fuzzy read, we always
  738. * have to visit all online cpus and make sum. So, for now, unnecessary
  739. * synchronization is not implemented. (just implemented for cpu hotplug)
  740. *
  741. * If there are kernel internal actions which can make use of some not-exact
  742. * value, and reading all cpu value can be performance bottleneck in some
  743. * common workload, threashold and synchonization as vmstat[] should be
  744. * implemented.
  745. */
  746. static long mem_cgroup_read_stat(struct mem_cgroup *memcg,
  747. enum mem_cgroup_stat_index idx)
  748. {
  749. long val = 0;
  750. int cpu;
  751. get_online_cpus();
  752. for_each_online_cpu(cpu)
  753. val += per_cpu(memcg->stat->count[idx], cpu);
  754. #ifdef CONFIG_HOTPLUG_CPU
  755. spin_lock(&memcg->pcp_counter_lock);
  756. val += memcg->nocpu_base.count[idx];
  757. spin_unlock(&memcg->pcp_counter_lock);
  758. #endif
  759. put_online_cpus();
  760. return val;
  761. }
  762. static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,
  763. bool charge)
  764. {
  765. int val = (charge) ? 1 : -1;
  766. this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SWAP], val);
  767. }
  768. static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg,
  769. enum mem_cgroup_events_index idx)
  770. {
  771. unsigned long val = 0;
  772. int cpu;
  773. get_online_cpus();
  774. for_each_online_cpu(cpu)
  775. val += per_cpu(memcg->stat->events[idx], cpu);
  776. #ifdef CONFIG_HOTPLUG_CPU
  777. spin_lock(&memcg->pcp_counter_lock);
  778. val += memcg->nocpu_base.events[idx];
  779. spin_unlock(&memcg->pcp_counter_lock);
  780. #endif
  781. put_online_cpus();
  782. return val;
  783. }
  784. static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
  785. struct page *page,
  786. bool anon, int nr_pages)
  787. {
  788. preempt_disable();
  789. /*
  790. * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
  791. * counted as CACHE even if it's on ANON LRU.
  792. */
  793. if (anon)
  794. __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS],
  795. nr_pages);
  796. else
  797. __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE],
  798. nr_pages);
  799. if (PageTransHuge(page))
  800. __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE],
  801. nr_pages);
  802. /* pagein of a big page is an event. So, ignore page size */
  803. if (nr_pages > 0)
  804. __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGIN]);
  805. else {
  806. __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]);
  807. nr_pages = -nr_pages; /* for event */
  808. }
  809. __this_cpu_add(memcg->stat->nr_page_events, nr_pages);
  810. preempt_enable();
  811. }
  812. unsigned long
  813. mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
  814. {
  815. struct mem_cgroup_per_zone *mz;
  816. mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
  817. return mz->lru_size[lru];
  818. }
  819. static unsigned long
  820. mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,
  821. unsigned int lru_mask)
  822. {
  823. struct mem_cgroup_per_zone *mz;
  824. enum lru_list lru;
  825. unsigned long ret = 0;
  826. mz = mem_cgroup_zoneinfo(memcg, nid, zid);
  827. for_each_lru(lru) {
  828. if (BIT(lru) & lru_mask)
  829. ret += mz->lru_size[lru];
  830. }
  831. return ret;
  832. }
  833. static unsigned long
  834. mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
  835. int nid, unsigned int lru_mask)
  836. {
  837. u64 total = 0;
  838. int zid;
  839. for (zid = 0; zid < MAX_NR_ZONES; zid++)
  840. total += mem_cgroup_zone_nr_lru_pages(memcg,
  841. nid, zid, lru_mask);
  842. return total;
  843. }
  844. static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
  845. unsigned int lru_mask)
  846. {
  847. int nid;
  848. u64 total = 0;
  849. for_each_node_state(nid, N_MEMORY)
  850. total += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
  851. return total;
  852. }
  853. static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
  854. enum mem_cgroup_events_target target)
  855. {
  856. unsigned long val, next;
  857. val = __this_cpu_read(memcg->stat->nr_page_events);
  858. next = __this_cpu_read(memcg->stat->targets[target]);
  859. /* from time_after() in jiffies.h */
  860. if ((long)next - (long)val < 0) {
  861. switch (target) {
  862. case MEM_CGROUP_TARGET_THRESH:
  863. next = val + THRESHOLDS_EVENTS_TARGET;
  864. break;
  865. case MEM_CGROUP_TARGET_SOFTLIMIT:
  866. next = val + SOFTLIMIT_EVENTS_TARGET;
  867. break;
  868. case MEM_CGROUP_TARGET_NUMAINFO:
  869. next = val + NUMAINFO_EVENTS_TARGET;
  870. break;
  871. default:
  872. break;
  873. }
  874. __this_cpu_write(memcg->stat->targets[target], next);
  875. return true;
  876. }
  877. return false;
  878. }
  879. /*
  880. * Check events in order.
  881. *
  882. */
  883. static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
  884. {
  885. preempt_disable();
  886. /* threshold event is triggered in finer grain than soft limit */
  887. if (unlikely(mem_cgroup_event_ratelimit(memcg,
  888. MEM_CGROUP_TARGET_THRESH))) {
  889. bool do_softlimit;
  890. bool do_numainfo __maybe_unused;
  891. do_softlimit = mem_cgroup_event_ratelimit(memcg,
  892. MEM_CGROUP_TARGET_SOFTLIMIT);
  893. #if MAX_NUMNODES > 1
  894. do_numainfo = mem_cgroup_event_ratelimit(memcg,
  895. MEM_CGROUP_TARGET_NUMAINFO);
  896. #endif
  897. preempt_enable();
  898. mem_cgroup_threshold(memcg);
  899. if (unlikely(do_softlimit))
  900. mem_cgroup_update_tree(memcg, page);
  901. #if MAX_NUMNODES > 1
  902. if (unlikely(do_numainfo))
  903. atomic_inc(&memcg->numainfo_events);
  904. #endif
  905. } else
  906. preempt_enable();
  907. }
  908. struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
  909. {
  910. /*
  911. * mm_update_next_owner() may clear mm->owner to NULL
  912. * if it races with swapoff, page migration, etc.
  913. * So this can be called with p == NULL.
  914. */
  915. if (unlikely(!p))
  916. return NULL;
  917. return mem_cgroup_from_css(task_css(p, mem_cgroup_subsys_id));
  918. }
  919. struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
  920. {
  921. struct mem_cgroup *memcg = NULL;
  922. if (!mm)
  923. return NULL;
  924. /*
  925. * Because we have no locks, mm->owner's may be being moved to other
  926. * cgroup. We use css_tryget() here even if this looks
  927. * pessimistic (rather than adding locks here).
  928. */
  929. rcu_read_lock();
  930. do {
  931. memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
  932. if (unlikely(!memcg))
  933. break;
  934. } while (!css_tryget(&memcg->css));
  935. rcu_read_unlock();
  936. return memcg;
  937. }
  938. /*
  939. * Returns a next (in a pre-order walk) alive memcg (with elevated css
  940. * ref. count) or NULL if the whole root's subtree has been visited.
  941. *
  942. * helper function to be used by mem_cgroup_iter
  943. */
  944. static struct mem_cgroup *__mem_cgroup_iter_next(struct mem_cgroup *root,
  945. struct mem_cgroup *last_visited)
  946. {
  947. struct cgroup_subsys_state *prev_css, *next_css;
  948. prev_css = last_visited ? &last_visited->css : NULL;
  949. skip_node:
  950. next_css = css_next_descendant_pre(prev_css, &root->css);
  951. /*
  952. * Even if we found a group we have to make sure it is
  953. * alive. css && !memcg means that the groups should be
  954. * skipped and we should continue the tree walk.
  955. * last_visited css is safe to use because it is
  956. * protected by css_get and the tree walk is rcu safe.
  957. */
  958. if (next_css) {
  959. struct mem_cgroup *mem = mem_cgroup_from_css(next_css);
  960. if (css_tryget(&mem->css))
  961. return mem;
  962. else {
  963. prev_css = next_css;
  964. goto skip_node;
  965. }
  966. }
  967. return NULL;
  968. }
  969. static void mem_cgroup_iter_invalidate(struct mem_cgroup *root)
  970. {
  971. /*
  972. * When a group in the hierarchy below root is destroyed, the
  973. * hierarchy iterator can no longer be trusted since it might
  974. * have pointed to the destroyed group. Invalidate it.
  975. */
  976. atomic_inc(&root->dead_count);
  977. }
  978. static struct mem_cgroup *
  979. mem_cgroup_iter_load(struct mem_cgroup_reclaim_iter *iter,
  980. struct mem_cgroup *root,
  981. int *sequence)
  982. {
  983. struct mem_cgroup *position = NULL;
  984. /*
  985. * A cgroup destruction happens in two stages: offlining and
  986. * release. They are separated by a RCU grace period.
  987. *
  988. * If the iterator is valid, we may still race with an
  989. * offlining. The RCU lock ensures the object won't be
  990. * released, tryget will fail if we lost the race.
  991. */
  992. *sequence = atomic_read(&root->dead_count);
  993. if (iter->last_dead_count == *sequence) {
  994. smp_rmb();
  995. position = iter->last_visited;
  996. if (position && !css_tryget(&position->css))
  997. position = NULL;
  998. }
  999. return position;
  1000. }
  1001. static void mem_cgroup_iter_update(struct mem_cgroup_reclaim_iter *iter,
  1002. struct mem_cgroup *last_visited,
  1003. struct mem_cgroup *new_position,
  1004. int sequence)
  1005. {
  1006. if (last_visited)
  1007. css_put(&last_visited->css);
  1008. /*
  1009. * We store the sequence count from the time @last_visited was
  1010. * loaded successfully instead of rereading it here so that we
  1011. * don't lose destruction events in between. We could have
  1012. * raced with the destruction of @new_position after all.
  1013. */
  1014. iter->last_visited = new_position;
  1015. smp_wmb();
  1016. iter->last_dead_count = sequence;
  1017. }
  1018. /**
  1019. * mem_cgroup_iter - iterate over memory cgroup hierarchy
  1020. * @root: hierarchy root
  1021. * @prev: previously returned memcg, NULL on first invocation
  1022. * @reclaim: cookie for shared reclaim walks, NULL for full walks
  1023. *
  1024. * Returns references to children of the hierarchy below @root, or
  1025. * @root itself, or %NULL after a full round-trip.
  1026. *
  1027. * Caller must pass the return value in @prev on subsequent
  1028. * invocations for reference counting, or use mem_cgroup_iter_break()
  1029. * to cancel a hierarchy walk before the round-trip is complete.
  1030. *
  1031. * Reclaimers can specify a zone and a priority level in @reclaim to
  1032. * divide up the memcgs in the hierarchy among all concurrent
  1033. * reclaimers operating on the same zone and priority.
  1034. */
  1035. struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
  1036. struct mem_cgroup *prev,
  1037. struct mem_cgroup_reclaim_cookie *reclaim)
  1038. {
  1039. struct mem_cgroup *memcg = NULL;
  1040. struct mem_cgroup *last_visited = NULL;
  1041. if (mem_cgroup_disabled())
  1042. return NULL;
  1043. if (!root)
  1044. root = root_mem_cgroup;
  1045. if (prev && !reclaim)
  1046. last_visited = prev;
  1047. if (!root->use_hierarchy && root != root_mem_cgroup) {
  1048. if (prev)
  1049. goto out_css_put;
  1050. return root;
  1051. }
  1052. rcu_read_lock();
  1053. while (!memcg) {
  1054. struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
  1055. int uninitialized_var(seq);
  1056. if (reclaim) {
  1057. int nid = zone_to_nid(reclaim->zone);
  1058. int zid = zone_idx(reclaim->zone);
  1059. struct mem_cgroup_per_zone *mz;
  1060. mz = mem_cgroup_zoneinfo(root, nid, zid);
  1061. iter = &mz->reclaim_iter[reclaim->priority];
  1062. if (prev && reclaim->generation != iter->generation) {
  1063. iter->last_visited = NULL;
  1064. goto out_unlock;
  1065. }
  1066. last_visited = mem_cgroup_iter_load(iter, root, &seq);
  1067. }
  1068. memcg = __mem_cgroup_iter_next(root, last_visited);
  1069. if (reclaim) {
  1070. mem_cgroup_iter_update(iter, last_visited, memcg, seq);
  1071. if (!memcg)
  1072. iter->generation++;
  1073. else if (!prev && memcg)
  1074. reclaim->generation = iter->generation;
  1075. }
  1076. if (prev && !memcg)
  1077. goto out_unlock;
  1078. }
  1079. out_unlock:
  1080. rcu_read_unlock();
  1081. out_css_put:
  1082. if (prev && prev != root)
  1083. css_put(&prev->css);
  1084. return memcg;
  1085. }
  1086. /**
  1087. * mem_cgroup_iter_break - abort a hierarchy walk prematurely
  1088. * @root: hierarchy root
  1089. * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
  1090. */
  1091. void mem_cgroup_iter_break(struct mem_cgroup *root,
  1092. struct mem_cgroup *prev)
  1093. {
  1094. if (!root)
  1095. root = root_mem_cgroup;
  1096. if (prev && prev != root)
  1097. css_put(&prev->css);
  1098. }
  1099. /*
  1100. * Iteration constructs for visiting all cgroups (under a tree). If
  1101. * loops are exited prematurely (break), mem_cgroup_iter_break() must
  1102. * be used for reference counting.
  1103. */
  1104. #define for_each_mem_cgroup_tree(iter, root) \
  1105. for (iter = mem_cgroup_iter(root, NULL, NULL); \
  1106. iter != NULL; \
  1107. iter = mem_cgroup_iter(root, iter, NULL))
  1108. #define for_each_mem_cgroup(iter) \
  1109. for (iter = mem_cgroup_iter(NULL, NULL, NULL); \
  1110. iter != NULL; \
  1111. iter = mem_cgroup_iter(NULL, iter, NULL))
  1112. void __mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx)
  1113. {
  1114. struct mem_cgroup *memcg;
  1115. rcu_read_lock();
  1116. memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
  1117. if (unlikely(!memcg))
  1118. goto out;
  1119. switch (idx) {
  1120. case PGFAULT:
  1121. this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGFAULT]);
  1122. break;
  1123. case PGMAJFAULT:
  1124. this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
  1125. break;
  1126. default:
  1127. BUG();
  1128. }
  1129. out:
  1130. rcu_read_unlock();
  1131. }
  1132. EXPORT_SYMBOL(__mem_cgroup_count_vm_event);
  1133. /**
  1134. * mem_cgroup_zone_lruvec - get the lru list vector for a zone and memcg
  1135. * @zone: zone of the wanted lruvec
  1136. * @memcg: memcg of the wanted lruvec
  1137. *
  1138. * Returns the lru list vector holding pages for the given @zone and
  1139. * @mem. This can be the global zone lruvec, if the memory controller
  1140. * is disabled.
  1141. */
  1142. struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
  1143. struct mem_cgroup *memcg)
  1144. {
  1145. struct mem_cgroup_per_zone *mz;
  1146. struct lruvec *lruvec;
  1147. if (mem_cgroup_disabled()) {
  1148. lruvec = &zone->lruvec;
  1149. goto out;
  1150. }
  1151. mz = mem_cgroup_zoneinfo(memcg, zone_to_nid(zone), zone_idx(zone));
  1152. lruvec = &mz->lruvec;
  1153. out:
  1154. /*
  1155. * Since a node can be onlined after the mem_cgroup was created,
  1156. * we have to be prepared to initialize lruvec->zone here;
  1157. * and if offlined then reonlined, we need to reinitialize it.
  1158. */
  1159. if (unlikely(lruvec->zone != zone))
  1160. lruvec->zone = zone;
  1161. return lruvec;
  1162. }
  1163. /*
  1164. * Following LRU functions are allowed to be used without PCG_LOCK.
  1165. * Operations are called by routine of global LRU independently from memcg.
  1166. * What we have to take care of here is validness of pc->mem_cgroup.
  1167. *
  1168. * Changes to pc->mem_cgroup happens when
  1169. * 1. charge
  1170. * 2. moving account
  1171. * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
  1172. * It is added to LRU before charge.
  1173. * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
  1174. * When moving account, the page is not on LRU. It's isolated.
  1175. */
  1176. /**
  1177. * mem_cgroup_page_lruvec - return lruvec for adding an lru page
  1178. * @page: the page
  1179. * @zone: zone of the page
  1180. */
  1181. struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct zone *zone)
  1182. {
  1183. struct mem_cgroup_per_zone *mz;
  1184. struct mem_cgroup *memcg;
  1185. struct page_cgroup *pc;
  1186. struct lruvec *lruvec;
  1187. if (mem_cgroup_disabled()) {
  1188. lruvec = &zone->lruvec;
  1189. goto out;
  1190. }
  1191. pc = lookup_page_cgroup(page);
  1192. memcg = pc->mem_cgroup;
  1193. /*
  1194. * Surreptitiously switch any uncharged offlist page to root:
  1195. * an uncharged page off lru does nothing to secure
  1196. * its former mem_cgroup from sudden removal.
  1197. *
  1198. * Our caller holds lru_lock, and PageCgroupUsed is updated
  1199. * under page_cgroup lock: between them, they make all uses
  1200. * of pc->mem_cgroup safe.
  1201. */
  1202. if (!PageLRU(page) && !PageCgroupUsed(pc) && memcg != root_mem_cgroup)
  1203. pc->mem_cgroup = memcg = root_mem_cgroup;
  1204. mz = page_cgroup_zoneinfo(memcg, page);
  1205. lruvec = &mz->lruvec;
  1206. out:
  1207. /*
  1208. * Since a node can be onlined after the mem_cgroup was created,
  1209. * we have to be prepared to initialize lruvec->zone here;
  1210. * and if offlined then reonlined, we need to reinitialize it.
  1211. */
  1212. if (unlikely(lruvec->zone != zone))
  1213. lruvec->zone = zone;
  1214. return lruvec;
  1215. }
  1216. /**
  1217. * mem_cgroup_update_lru_size - account for adding or removing an lru page
  1218. * @lruvec: mem_cgroup per zone lru vector
  1219. * @lru: index of lru list the page is sitting on
  1220. * @nr_pages: positive when adding or negative when removing
  1221. *
  1222. * This function must be called when a page is added to or removed from an
  1223. * lru list.
  1224. */
  1225. void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
  1226. int nr_pages)
  1227. {
  1228. struct mem_cgroup_per_zone *mz;
  1229. unsigned long *lru_size;
  1230. if (mem_cgroup_disabled())
  1231. return;
  1232. mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
  1233. lru_size = mz->lru_size + lru;
  1234. *lru_size += nr_pages;
  1235. VM_BUG_ON((long)(*lru_size) < 0);
  1236. }
  1237. /*
  1238. * Checks whether given mem is same or in the root_mem_cgroup's
  1239. * hierarchy subtree
  1240. */
  1241. bool __mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
  1242. struct mem_cgroup *memcg)
  1243. {
  1244. if (root_memcg == memcg)
  1245. return true;
  1246. if (!root_memcg->use_hierarchy || !memcg)
  1247. return false;
  1248. return cgroup_is_descendant(memcg->css.cgroup, root_memcg->css.cgroup);
  1249. }
  1250. static bool mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
  1251. struct mem_cgroup *memcg)
  1252. {
  1253. bool ret;
  1254. rcu_read_lock();
  1255. ret = __mem_cgroup_same_or_subtree(root_memcg, memcg);
  1256. rcu_read_unlock();
  1257. return ret;
  1258. }
  1259. bool task_in_mem_cgroup(struct task_struct *task,
  1260. const struct mem_cgroup *memcg)
  1261. {
  1262. struct mem_cgroup *curr = NULL;
  1263. struct task_struct *p;
  1264. bool ret;
  1265. p = find_lock_task_mm(task);
  1266. if (p) {
  1267. curr = try_get_mem_cgroup_from_mm(p->mm);
  1268. task_unlock(p);
  1269. } else {
  1270. /*
  1271. * All threads may have already detached their mm's, but the oom
  1272. * killer still needs to detect if they have already been oom
  1273. * killed to prevent needlessly killing additional tasks.
  1274. */
  1275. rcu_read_lock();
  1276. curr = mem_cgroup_from_task(task);
  1277. if (curr)
  1278. css_get(&curr->css);
  1279. rcu_read_unlock();
  1280. }
  1281. if (!curr)
  1282. return false;
  1283. /*
  1284. * We should check use_hierarchy of "memcg" not "curr". Because checking
  1285. * use_hierarchy of "curr" here make this function true if hierarchy is
  1286. * enabled in "curr" and "curr" is a child of "memcg" in *cgroup*
  1287. * hierarchy(even if use_hierarchy is disabled in "memcg").
  1288. */
  1289. ret = mem_cgroup_same_or_subtree(memcg, curr);
  1290. css_put(&curr->css);
  1291. return ret;
  1292. }
  1293. int mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec)
  1294. {
  1295. unsigned long inactive_ratio;
  1296. unsigned long inactive;
  1297. unsigned long active;
  1298. unsigned long gb;
  1299. inactive = mem_cgroup_get_lru_size(lruvec, LRU_INACTIVE_ANON);
  1300. active = mem_cgroup_get_lru_size(lruvec, LRU_ACTIVE_ANON);
  1301. gb = (inactive + active) >> (30 - PAGE_SHIFT);
  1302. if (gb)
  1303. inactive_ratio = int_sqrt(10 * gb);
  1304. else
  1305. inactive_ratio = 1;
  1306. return inactive * inactive_ratio < active;
  1307. }
  1308. #define mem_cgroup_from_res_counter(counter, member) \
  1309. container_of(counter, struct mem_cgroup, member)
  1310. /**
  1311. * mem_cgroup_margin - calculate chargeable space of a memory cgroup
  1312. * @memcg: the memory cgroup
  1313. *
  1314. * Returns the maximum amount of memory @mem can be charged with, in
  1315. * pages.
  1316. */
  1317. static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
  1318. {
  1319. unsigned long long margin;
  1320. margin = res_counter_margin(&memcg->res);
  1321. if (do_swap_account)
  1322. margin = min(margin, res_counter_margin(&memcg->memsw));
  1323. return margin >> PAGE_SHIFT;
  1324. }
  1325. int mem_cgroup_swappiness(struct mem_cgroup *memcg)
  1326. {
  1327. /* root ? */
  1328. if (!css_parent(&memcg->css))
  1329. return vm_swappiness;
  1330. return memcg->swappiness;
  1331. }
  1332. /*
  1333. * memcg->moving_account is used for checking possibility that some thread is
  1334. * calling move_account(). When a thread on CPU-A starts moving pages under
  1335. * a memcg, other threads should check memcg->moving_account under
  1336. * rcu_read_lock(), like this:
  1337. *
  1338. * CPU-A CPU-B
  1339. * rcu_read_lock()
  1340. * memcg->moving_account+1 if (memcg->mocing_account)
  1341. * take heavy locks.
  1342. * synchronize_rcu() update something.
  1343. * rcu_read_unlock()
  1344. * start move here.
  1345. */
  1346. /* for quick checking without looking up memcg */
  1347. atomic_t memcg_moving __read_mostly;
  1348. static void mem_cgroup_start_move(struct mem_cgroup *memcg)
  1349. {
  1350. atomic_inc(&memcg_moving);
  1351. atomic_inc(&memcg->moving_account);
  1352. synchronize_rcu();
  1353. }
  1354. static void mem_cgroup_end_move(struct mem_cgroup *memcg)
  1355. {
  1356. /*
  1357. * Now, mem_cgroup_clear_mc() may call this function with NULL.
  1358. * We check NULL in callee rather than caller.
  1359. */
  1360. if (memcg) {
  1361. atomic_dec(&memcg_moving);
  1362. atomic_dec(&memcg->moving_account);
  1363. }
  1364. }
  1365. /*
  1366. * 2 routines for checking "mem" is under move_account() or not.
  1367. *
  1368. * mem_cgroup_stolen() - checking whether a cgroup is mc.from or not. This
  1369. * is used for avoiding races in accounting. If true,
  1370. * pc->mem_cgroup may be overwritten.
  1371. *
  1372. * mem_cgroup_under_move() - checking a cgroup is mc.from or mc.to or
  1373. * under hierarchy of moving cgroups. This is for
  1374. * waiting at hith-memory prressure caused by "move".
  1375. */
  1376. static bool mem_cgroup_stolen(struct mem_cgroup *memcg)
  1377. {
  1378. VM_BUG_ON(!rcu_read_lock_held());
  1379. return atomic_read(&memcg->moving_account) > 0;
  1380. }
  1381. static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
  1382. {
  1383. struct mem_cgroup *from;
  1384. struct mem_cgroup *to;
  1385. bool ret = false;
  1386. /*
  1387. * Unlike task_move routines, we access mc.to, mc.from not under
  1388. * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
  1389. */
  1390. spin_lock(&mc.lock);
  1391. from = mc.from;
  1392. to = mc.to;
  1393. if (!from)
  1394. goto unlock;
  1395. ret = mem_cgroup_same_or_subtree(memcg, from)
  1396. || mem_cgroup_same_or_subtree(memcg, to);
  1397. unlock:
  1398. spin_unlock(&mc.lock);
  1399. return ret;
  1400. }
  1401. static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
  1402. {
  1403. if (mc.moving_task && current != mc.moving_task) {
  1404. if (mem_cgroup_under_move(memcg)) {
  1405. DEFINE_WAIT(wait);
  1406. prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
  1407. /* moving charge context might have finished. */
  1408. if (mc.moving_task)
  1409. schedule();
  1410. finish_wait(&mc.waitq, &wait);
  1411. return true;
  1412. }
  1413. }
  1414. return false;
  1415. }
  1416. /*
  1417. * Take this lock when
  1418. * - a code tries to modify page's memcg while it's USED.
  1419. * - a code tries to modify page state accounting in a memcg.
  1420. * see mem_cgroup_stolen(), too.
  1421. */
  1422. static void move_lock_mem_cgroup(struct mem_cgroup *memcg,
  1423. unsigned long *flags)
  1424. {
  1425. spin_lock_irqsave(&memcg->move_lock, *flags);
  1426. }
  1427. static void move_unlock_mem_cgroup(struct mem_cgroup *memcg,
  1428. unsigned long *flags)
  1429. {
  1430. spin_unlock_irqrestore(&memcg->move_lock, *flags);
  1431. }
  1432. #define K(x) ((x) << (PAGE_SHIFT-10))
  1433. /**
  1434. * mem_cgroup_print_oom_info: Print OOM information relevant to memory controller.
  1435. * @memcg: The memory cgroup that went over limit
  1436. * @p: Task that is going to be killed
  1437. *
  1438. * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
  1439. * enabled
  1440. */
  1441. void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
  1442. {
  1443. struct cgroup *task_cgrp;
  1444. struct cgroup *mem_cgrp;
  1445. /*
  1446. * Need a buffer in BSS, can't rely on allocations. The code relies
  1447. * on the assumption that OOM is serialized for memory controller.
  1448. * If this assumption is broken, revisit this code.
  1449. */
  1450. static char memcg_name[PATH_MAX];
  1451. int ret;
  1452. struct mem_cgroup *iter;
  1453. unsigned int i;
  1454. if (!p)
  1455. return;
  1456. rcu_read_lock();
  1457. mem_cgrp = memcg->css.cgroup;
  1458. task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);
  1459. ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
  1460. if (ret < 0) {
  1461. /*
  1462. * Unfortunately, we are unable to convert to a useful name
  1463. * But we'll still print out the usage information
  1464. */
  1465. rcu_read_unlock();
  1466. goto done;
  1467. }
  1468. rcu_read_unlock();
  1469. pr_info("Task in %s killed", memcg_name);
  1470. rcu_read_lock();
  1471. ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
  1472. if (ret < 0) {
  1473. rcu_read_unlock();
  1474. goto done;
  1475. }
  1476. rcu_read_unlock();
  1477. /*
  1478. * Continues from above, so we don't need an KERN_ level
  1479. */
  1480. pr_cont(" as a result of limit of %s\n", memcg_name);
  1481. done:
  1482. pr_info("memory: usage %llukB, limit %llukB, failcnt %llu\n",
  1483. res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
  1484. res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
  1485. res_counter_read_u64(&memcg->res, RES_FAILCNT));
  1486. pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %llu\n",
  1487. res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
  1488. res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
  1489. res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
  1490. pr_info("kmem: usage %llukB, limit %llukB, failcnt %llu\n",
  1491. res_counter_read_u64(&memcg->kmem, RES_USAGE) >> 10,
  1492. res_counter_read_u64(&memcg->kmem, RES_LIMIT) >> 10,
  1493. res_counter_read_u64(&memcg->kmem, RES_FAILCNT));
  1494. for_each_mem_cgroup_tree(iter, memcg) {
  1495. pr_info("Memory cgroup stats");
  1496. rcu_read_lock();
  1497. ret = cgroup_path(iter->css.cgroup, memcg_name, PATH_MAX);
  1498. if (!ret)
  1499. pr_cont(" for %s", memcg_name);
  1500. rcu_read_unlock();
  1501. pr_cont(":");
  1502. for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
  1503. if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
  1504. continue;
  1505. pr_cont(" %s:%ldKB", mem_cgroup_stat_names[i],
  1506. K(mem_cgroup_read_stat(iter, i)));
  1507. }
  1508. for (i = 0; i < NR_LRU_LISTS; i++)
  1509. pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
  1510. K(mem_cgroup_nr_lru_pages(iter, BIT(i))));
  1511. pr_cont("\n");
  1512. }
  1513. }
  1514. /*
  1515. * This function returns the number of memcg under hierarchy tree. Returns
  1516. * 1(self count) if no children.
  1517. */
  1518. static int mem_cgroup_count_children(struct mem_cgroup *memcg)
  1519. {
  1520. int num = 0;
  1521. struct mem_cgroup *iter;
  1522. for_each_mem_cgroup_tree(iter, memcg)
  1523. num++;
  1524. return num;
  1525. }
  1526. /*
  1527. * Return the memory (and swap, if configured) limit for a memcg.
  1528. */
  1529. static u64 mem_cgroup_get_limit(struct mem_cgroup *memcg)
  1530. {
  1531. u64 limit;
  1532. limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
  1533. /*
  1534. * Do not consider swap space if we cannot swap due to swappiness
  1535. */
  1536. if (mem_cgroup_swappiness(memcg)) {
  1537. u64 memsw;
  1538. limit += total_swap_pages << PAGE_SHIFT;
  1539. memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
  1540. /*
  1541. * If memsw is finite and limits the amount of swap space
  1542. * available to this memcg, return that limit.
  1543. */
  1544. limit = min(limit, memsw);
  1545. }
  1546. return limit;
  1547. }
  1548. static void mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
  1549. int order)
  1550. {
  1551. struct mem_cgroup *iter;
  1552. unsigned long chosen_points = 0;
  1553. unsigned long totalpages;
  1554. unsigned int points = 0;
  1555. struct task_struct *chosen = NULL;
  1556. /*
  1557. * If current has a pending SIGKILL or is exiting, then automatically
  1558. * select it. The goal is to allow it to allocate so that it may
  1559. * quickly exit and free its memory.
  1560. */
  1561. if (fatal_signal_pending(current) || current->flags & PF_EXITING) {
  1562. set_thread_flag(TIF_MEMDIE);
  1563. return;
  1564. }
  1565. check_panic_on_oom(CONSTRAINT_MEMCG, gfp_mask, order, NULL);
  1566. totalpages = mem_cgroup_get_limit(memcg) >> PAGE_SHIFT ? : 1;
  1567. for_each_mem_cgroup_tree(iter, memcg) {
  1568. struct css_task_iter it;
  1569. struct task_struct *task;
  1570. css_task_iter_start(&iter->css, &it);
  1571. while ((task = css_task_iter_next(&it))) {
  1572. switch (oom_scan_process_thread(task, totalpages, NULL,
  1573. false)) {
  1574. case OOM_SCAN_SELECT:
  1575. if (chosen)
  1576. put_task_struct(chosen);
  1577. chosen = task;
  1578. chosen_points = ULONG_MAX;
  1579. get_task_struct(chosen);
  1580. /* fall through */
  1581. case OOM_SCAN_CONTINUE:
  1582. continue;
  1583. case OOM_SCAN_ABORT:
  1584. css_task_iter_end(&it);
  1585. mem_cgroup_iter_break(memcg, iter);
  1586. if (chosen)
  1587. put_task_struct(chosen);
  1588. return;
  1589. case OOM_SCAN_OK:
  1590. break;
  1591. };
  1592. points = oom_badness(task, memcg, NULL, totalpages);
  1593. if (points > chosen_points) {
  1594. if (chosen)
  1595. put_task_struct(chosen);
  1596. chosen = task;
  1597. chosen_points = points;
  1598. get_task_struct(chosen);
  1599. }
  1600. }
  1601. css_task_iter_end(&it);
  1602. }
  1603. if (!chosen)
  1604. return;
  1605. points = chosen_points * 1000 / totalpages;
  1606. oom_kill_process(chosen, gfp_mask, order, points, totalpages, memcg,
  1607. NULL, "Memory cgroup out of memory");
  1608. }
  1609. static unsigned long mem_cgroup_reclaim(struct mem_cgroup *memcg,
  1610. gfp_t gfp_mask,
  1611. unsigned long flags)
  1612. {
  1613. unsigned long total = 0;
  1614. bool noswap = false;
  1615. int loop;
  1616. if (flags & MEM_CGROUP_RECLAIM_NOSWAP)
  1617. noswap = true;
  1618. if (!(flags & MEM_CGROUP_RECLAIM_SHRINK) && memcg->memsw_is_minimum)
  1619. noswap = true;
  1620. for (loop = 0; loop < MEM_CGROUP_MAX_RECLAIM_LOOPS; loop++) {
  1621. if (loop)
  1622. drain_all_stock_async(memcg);
  1623. total += try_to_free_mem_cgroup_pages(memcg, gfp_mask, noswap);
  1624. /*
  1625. * Allow limit shrinkers, which are triggered directly
  1626. * by userspace, to catch signals and stop reclaim
  1627. * after minimal progress, regardless of the margin.
  1628. */
  1629. if (total && (flags & MEM_CGROUP_RECLAIM_SHRINK))
  1630. break;
  1631. if (mem_cgroup_margin(memcg))
  1632. break;
  1633. /*
  1634. * If nothing was reclaimed after two attempts, there
  1635. * may be no reclaimable pages in this hierarchy.
  1636. */
  1637. if (loop && !total)
  1638. break;
  1639. }
  1640. return total;
  1641. }
  1642. /**
  1643. * test_mem_cgroup_node_reclaimable
  1644. * @memcg: the target memcg
  1645. * @nid: the node ID to be checked.
  1646. * @noswap : specify true here if the user wants flle only information.
  1647. *
  1648. * This function returns whether the specified memcg contains any
  1649. * reclaimable pages on a node. Returns true if there are any reclaimable
  1650. * pages in the node.
  1651. */
  1652. static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
  1653. int nid, bool noswap)
  1654. {
  1655. if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
  1656. return true;
  1657. if (noswap || !total_swap_pages)
  1658. return false;
  1659. if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
  1660. return true;
  1661. return false;
  1662. }
  1663. #if MAX_NUMNODES > 1
  1664. /*
  1665. * Always updating the nodemask is not very good - even if we have an empty
  1666. * list or the wrong list here, we can start from some node and traverse all
  1667. * nodes based on the zonelist. So update the list loosely once per 10 secs.
  1668. *
  1669. */
  1670. static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
  1671. {
  1672. int nid;
  1673. /*
  1674. * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
  1675. * pagein/pageout changes since the last update.
  1676. */
  1677. if (!atomic_read(&memcg->numainfo_events))
  1678. return;
  1679. if (atomic_inc_return(&memcg->numainfo_updating) > 1)
  1680. return;
  1681. /* make a nodemask where this memcg uses memory from */
  1682. memcg->scan_nodes = node_states[N_MEMORY];
  1683. for_each_node_mask(nid, node_states[N_MEMORY]) {
  1684. if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
  1685. node_clear(nid, memcg->scan_nodes);
  1686. }
  1687. atomic_set(&memcg->numainfo_events, 0);
  1688. atomic_set(&memcg->numainfo_updating, 0);
  1689. }
  1690. /*
  1691. * Selecting a node where we start reclaim from. Because what we need is just
  1692. * reducing usage counter, start from anywhere is O,K. Considering
  1693. * memory reclaim from current node, there are pros. and cons.
  1694. *
  1695. * Freeing memory from current node means freeing memory from a node which
  1696. * we'll use or we've used. So, it may make LRU bad. And if several threads
  1697. * hit limits, it will see a contention on a node. But freeing from remote
  1698. * node means more costs for memory reclaim because of memory latency.
  1699. *
  1700. * Now, we use round-robin. Better algorithm is welcomed.
  1701. */
  1702. int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
  1703. {
  1704. int node;
  1705. mem_cgroup_may_update_nodemask(memcg);
  1706. node = memcg->last_scanned_node;
  1707. node = next_node(node, memcg->scan_nodes);
  1708. if (node == MAX_NUMNODES)
  1709. node = first_node(memcg->scan_nodes);
  1710. /*
  1711. * We call this when we hit limit, not when pages are added to LRU.
  1712. * No LRU may hold pages because all pages are UNEVICTABLE or
  1713. * memcg is too small and all pages are not on LRU. In that case,
  1714. * we use curret node.
  1715. */
  1716. if (unlikely(node == MAX_NUMNODES))
  1717. node = numa_node_id();
  1718. memcg->last_scanned_node = node;
  1719. return node;
  1720. }
  1721. /*
  1722. * Check all nodes whether it contains reclaimable pages or not.
  1723. * For quick scan, we make use of scan_nodes. This will allow us to skip
  1724. * unused nodes. But scan_nodes is lazily updated and may not cotain
  1725. * enough new information. We need to do double check.
  1726. */
  1727. static bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
  1728. {
  1729. int nid;
  1730. /*
  1731. * quick check...making use of scan_node.
  1732. * We can skip unused nodes.
  1733. */
  1734. if (!nodes_empty(memcg->scan_nodes)) {
  1735. for (nid = first_node(memcg->scan_nodes);
  1736. nid < MAX_NUMNODES;
  1737. nid = next_node(nid, memcg->scan_nodes)) {
  1738. if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
  1739. return true;
  1740. }
  1741. }
  1742. /*
  1743. * Check rest of nodes.
  1744. */
  1745. for_each_node_state(nid, N_MEMORY) {
  1746. if (node_isset(nid, memcg->scan_nodes))
  1747. continue;
  1748. if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
  1749. return true;
  1750. }
  1751. return false;
  1752. }
  1753. #else
  1754. int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
  1755. {
  1756. return 0;
  1757. }
  1758. static bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
  1759. {
  1760. return test_mem_cgroup_node_reclaimable(memcg, 0, noswap);
  1761. }
  1762. #endif
  1763. static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
  1764. struct zone *zone,
  1765. gfp_t gfp_mask,
  1766. unsigned long *total_scanned)
  1767. {
  1768. struct mem_cgroup *victim = NULL;
  1769. int total = 0;
  1770. int loop = 0;
  1771. unsigned long excess;
  1772. unsigned long nr_scanned;
  1773. struct mem_cgroup_reclaim_cookie reclaim = {
  1774. .zone = zone,
  1775. .priority = 0,
  1776. };
  1777. excess = res_counter_soft_limit_excess(&root_memcg->res) >> PAGE_SHIFT;
  1778. while (1) {
  1779. victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
  1780. if (!victim) {
  1781. loop++;
  1782. if (loop >= 2) {
  1783. /*
  1784. * If we have not been able to reclaim
  1785. * anything, it might because there are
  1786. * no reclaimable pages under this hierarchy
  1787. */
  1788. if (!total)
  1789. break;
  1790. /*
  1791. * We want to do more targeted reclaim.
  1792. * excess >> 2 is not to excessive so as to
  1793. * reclaim too much, nor too less that we keep
  1794. * coming back to reclaim from this cgroup
  1795. */
  1796. if (total >= (excess >> 2) ||
  1797. (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
  1798. break;
  1799. }
  1800. continue;
  1801. }
  1802. if (!mem_cgroup_reclaimable(victim, false))
  1803. continue;
  1804. total += mem_cgroup_shrink_node_zone(victim, gfp_mask, false,
  1805. zone, &nr_scanned);
  1806. *total_scanned += nr_scanned;
  1807. if (!res_counter_soft_limit_excess(&root_memcg->res))
  1808. break;
  1809. }
  1810. mem_cgroup_iter_break(root_memcg, victim);
  1811. return total;
  1812. }
  1813. #ifdef CONFIG_LOCKDEP
  1814. static struct lockdep_map memcg_oom_lock_dep_map = {
  1815. .name = "memcg_oom_lock",
  1816. };
  1817. #endif
  1818. static DEFINE_SPINLOCK(memcg_oom_lock);
  1819. /*
  1820. * Check OOM-Killer is already running under our hierarchy.
  1821. * If someone is running, return false.
  1822. */
  1823. static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
  1824. {
  1825. struct mem_cgroup *iter, *failed = NULL;
  1826. spin_lock(&memcg_oom_lock);
  1827. for_each_mem_cgroup_tree(iter, memcg) {
  1828. if (iter->oom_lock) {
  1829. /*
  1830. * this subtree of our hierarchy is already locked
  1831. * so we cannot give a lock.
  1832. */
  1833. failed = iter;
  1834. mem_cgroup_iter_break(memcg, iter);
  1835. break;
  1836. } else
  1837. iter->oom_lock = true;
  1838. }
  1839. if (failed) {
  1840. /*
  1841. * OK, we failed to lock the whole subtree so we have
  1842. * to clean up what we set up to the failing subtree
  1843. */
  1844. for_each_mem_cgroup_tree(iter, memcg) {
  1845. if (iter == failed) {
  1846. mem_cgroup_iter_break(memcg, iter);
  1847. break;
  1848. }
  1849. iter->oom_lock = false;
  1850. }
  1851. } else
  1852. mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
  1853. spin_unlock(&memcg_oom_lock);
  1854. return !failed;
  1855. }
  1856. static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
  1857. {
  1858. struct mem_cgroup *iter;
  1859. spin_lock(&memcg_oom_lock);
  1860. mutex_release(&memcg_oom_lock_dep_map, 1, _RET_IP_);
  1861. for_each_mem_cgroup_tree(iter, memcg)
  1862. iter->oom_lock = false;
  1863. spin_unlock(&memcg_oom_lock);
  1864. }
  1865. static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
  1866. {
  1867. struct mem_cgroup *iter;
  1868. for_each_mem_cgroup_tree(iter, memcg)
  1869. atomic_inc(&iter->under_oom);
  1870. }
  1871. static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
  1872. {
  1873. struct mem_cgroup *iter;
  1874. /*
  1875. * When a new child is created while the hierarchy is under oom,
  1876. * mem_cgroup_oom_lock() may not be called. We have to use
  1877. * atomic_add_unless() here.
  1878. */
  1879. for_each_mem_cgroup_tree(iter, memcg)
  1880. atomic_add_unless(&iter->under_oom, -1, 0);
  1881. }
  1882. static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
  1883. struct oom_wait_info {
  1884. struct mem_cgroup *memcg;
  1885. wait_queue_t wait;
  1886. };
  1887. static int memcg_oom_wake_function(wait_queue_t *wait,
  1888. unsigned mode, int sync, void *arg)
  1889. {
  1890. struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
  1891. struct mem_cgroup *oom_wait_memcg;
  1892. struct oom_wait_info *oom_wait_info;
  1893. oom_wait_info = container_of(wait, struct oom_wait_info, wait);
  1894. oom_wait_memcg = oom_wait_info->memcg;
  1895. /*
  1896. * Both of oom_wait_info->memcg and wake_memcg are stable under us.
  1897. * Then we can use css_is_ancestor without taking care of RCU.
  1898. */
  1899. if (!mem_cgroup_same_or_subtree(oom_wait_memcg, wake_memcg)
  1900. && !mem_cgroup_same_or_subtree(wake_memcg, oom_wait_memcg))
  1901. return 0;
  1902. return autoremove_wake_function(wait, mode, sync, arg);
  1903. }
  1904. static void memcg_wakeup_oom(struct mem_cgroup *memcg)
  1905. {
  1906. atomic_inc(&memcg->oom_wakeups);
  1907. /* for filtering, pass "memcg" as argument. */
  1908. __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
  1909. }
  1910. static void memcg_oom_recover(struct mem_cgroup *memcg)
  1911. {
  1912. if (memcg && atomic_read(&memcg->under_oom))
  1913. memcg_wakeup_oom(memcg);
  1914. }
  1915. static void mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
  1916. {
  1917. if (!current->memcg_oom.may_oom)
  1918. return;
  1919. /*
  1920. * We are in the middle of the charge context here, so we
  1921. * don't want to block when potentially sitting on a callstack
  1922. * that holds all kinds of filesystem and mm locks.
  1923. *
  1924. * Also, the caller may handle a failed allocation gracefully
  1925. * (like optional page cache readahead) and so an OOM killer
  1926. * invocation might not even be necessary.
  1927. *
  1928. * That's why we don't do anything here except remember the
  1929. * OOM context and then deal with it at the end of the page
  1930. * fault when the stack is unwound, the locks are released,
  1931. * and when we know whether the fault was overall successful.
  1932. */
  1933. css_get(&memcg->css);
  1934. current->memcg_oom.memcg = memcg;
  1935. current->memcg_oom.gfp_mask = mask;
  1936. current->memcg_oom.order = order;
  1937. }
  1938. /**
  1939. * mem_cgroup_oom_synchronize - complete memcg OOM handling
  1940. * @handle: actually kill/wait or just clean up the OOM state
  1941. *
  1942. * This has to be called at the end of a page fault if the memcg OOM
  1943. * handler was enabled.
  1944. *
  1945. * Memcg supports userspace OOM handling where failed allocations must
  1946. * sleep on a waitqueue until the userspace task resolves the
  1947. * situation. Sleeping directly in the charge context with all kinds
  1948. * of locks held is not a good idea, instead we remember an OOM state
  1949. * in the task and mem_cgroup_oom_synchronize() has to be called at
  1950. * the end of the page fault to complete the OOM handling.
  1951. *
  1952. * Returns %true if an ongoing memcg OOM situation was detected and
  1953. * completed, %false otherwise.
  1954. */
  1955. bool mem_cgroup_oom_synchronize(bool handle)
  1956. {
  1957. struct mem_cgroup *memcg = current->memcg_oom.memcg;
  1958. struct oom_wait_info owait;
  1959. bool locked;
  1960. /* OOM is global, do not handle */
  1961. if (!memcg)
  1962. return false;
  1963. if (!handle)
  1964. goto cleanup;
  1965. owait.memcg = memcg;
  1966. owait.wait.flags = 0;
  1967. owait.wait.func = memcg_oom_wake_function;
  1968. owait.wait.private = current;
  1969. INIT_LIST_HEAD(&owait.wait.task_list);
  1970. prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
  1971. mem_cgroup_mark_under_oom(memcg);
  1972. locked = mem_cgroup_oom_trylock(memcg);
  1973. if (locked)
  1974. mem_cgroup_oom_notify(memcg);
  1975. if (locked && !memcg->oom_kill_disable) {
  1976. mem_cgroup_unmark_under_oom(memcg);
  1977. finish_wait(&memcg_oom_waitq, &owait.wait);
  1978. mem_cgroup_out_of_memory(memcg, current->memcg_oom.gfp_mask,
  1979. current->memcg_oom.order);
  1980. } else {
  1981. schedule();
  1982. mem_cgroup_unmark_under_oom(memcg);
  1983. finish_wait(&memcg_oom_waitq, &owait.wait);
  1984. }
  1985. if (locked) {
  1986. mem_cgroup_oom_unlock(memcg);
  1987. /*
  1988. * There is no guarantee that an OOM-lock contender
  1989. * sees the wakeups triggered by the OOM kill
  1990. * uncharges. Wake any sleepers explicitely.
  1991. */
  1992. memcg_oom_recover(memcg);
  1993. }
  1994. cleanup:
  1995. current->memcg_oom.memcg = NULL;
  1996. css_put(&memcg->css);
  1997. return true;
  1998. }
  1999. /*
  2000. * Currently used to update mapped file statistics, but the routine can be
  2001. * generalized to update other statistics as well.
  2002. *
  2003. * Notes: Race condition
  2004. *
  2005. * We usually use page_cgroup_lock() for accessing page_cgroup member but
  2006. * it tends to be costly. But considering some conditions, we doesn't need
  2007. * to do so _always_.
  2008. *
  2009. * Considering "charge", lock_page_cgroup() is not required because all
  2010. * file-stat operations happen after a page is attached to radix-tree. There
  2011. * are no race with "charge".
  2012. *
  2013. * Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup
  2014. * at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even
  2015. * if there are race with "uncharge". Statistics itself is properly handled
  2016. * by flags.
  2017. *
  2018. * Considering "move", this is an only case we see a race. To make the race
  2019. * small, we check mm->moving_account and detect there are possibility of race
  2020. * If there is, we take a lock.
  2021. */
  2022. void __mem_cgroup_begin_update_page_stat(struct page *page,
  2023. bool *locked, unsigned long *flags)
  2024. {
  2025. struct mem_cgroup *memcg;
  2026. struct page_cgroup *pc;
  2027. pc = lookup_page_cgroup(page);
  2028. again:
  2029. memcg = pc->mem_cgroup;
  2030. if (unlikely(!memcg || !PageCgroupUsed(pc)))
  2031. return;
  2032. /*
  2033. * If this memory cgroup is not under account moving, we don't
  2034. * need to take move_lock_mem_cgroup(). Because we already hold
  2035. * rcu_read_lock(), any calls to move_account will be delayed until
  2036. * rcu_read_unlock() if mem_cgroup_stolen() == true.
  2037. */
  2038. if (!mem_cgroup_stolen(memcg))
  2039. return;
  2040. move_lock_mem_cgroup(memcg, flags);
  2041. if (memcg != pc->mem_cgroup || !PageCgroupUsed(pc)) {
  2042. move_unlock_mem_cgroup(memcg, flags);
  2043. goto again;
  2044. }
  2045. *locked = true;
  2046. }
  2047. void __mem_cgroup_end_update_page_stat(struct page *page, unsigned long *flags)
  2048. {
  2049. struct page_cgroup *pc = lookup_page_cgroup(page);
  2050. /*
  2051. * It's guaranteed that pc->mem_cgroup never changes while
  2052. * lock is held because a routine modifies pc->mem_cgroup
  2053. * should take move_lock_mem_cgroup().
  2054. */
  2055. move_unlock_mem_cgroup(pc->mem_cgroup, flags);
  2056. }
  2057. void mem_cgroup_update_page_stat(struct page *page,
  2058. enum mem_cgroup_stat_index idx, int val)
  2059. {
  2060. struct mem_cgroup *memcg;
  2061. struct page_cgroup *pc = lookup_page_cgroup(page);
  2062. unsigned long uninitialized_var(flags);
  2063. if (mem_cgroup_disabled())
  2064. return;
  2065. VM_BUG_ON(!rcu_read_lock_held());
  2066. memcg = pc->mem_cgroup;
  2067. if (unlikely(!memcg || !PageCgroupUsed(pc)))
  2068. return;
  2069. this_cpu_add(memcg->stat->count[idx], val);
  2070. }
  2071. /*
  2072. * size of first charge trial. "32" comes from vmscan.c's magic value.
  2073. * TODO: maybe necessary to use big numbers in big irons.
  2074. */
  2075. #define CHARGE_BATCH 32U
  2076. struct memcg_stock_pcp {
  2077. struct mem_cgroup *cached; /* this never be root cgroup */
  2078. unsigned int nr_pages;
  2079. struct work_struct work;
  2080. unsigned long flags;
  2081. #define FLUSHING_CACHED_CHARGE 0
  2082. };
  2083. static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
  2084. static DEFINE_MUTEX(percpu_charge_mutex);
  2085. /**
  2086. * consume_stock: Try to consume stocked charge on this cpu.
  2087. * @memcg: memcg to consume from.
  2088. * @nr_pages: how many pages to charge.
  2089. *
  2090. * The charges will only happen if @memcg matches the current cpu's memcg
  2091. * stock, and at least @nr_pages are available in that stock. Failure to
  2092. * service an allocation will refill the stock.
  2093. *
  2094. * returns true if successful, false otherwise.
  2095. */
  2096. static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
  2097. {
  2098. struct memcg_stock_pcp *stock;
  2099. bool ret = true;
  2100. if (nr_pages > CHARGE_BATCH)
  2101. return false;
  2102. stock = &get_cpu_var(memcg_stock);
  2103. if (memcg == stock->cached && stock->nr_pages >= nr_pages)
  2104. stock->nr_pages -= nr_pages;
  2105. else /* need to call res_counter_charge */
  2106. ret = false;
  2107. put_cpu_var(memcg_stock);
  2108. return ret;
  2109. }
  2110. /*
  2111. * Returns stocks cached in percpu to res_counter and reset cached information.
  2112. */
  2113. static void drain_stock(struct memcg_stock_pcp *stock)
  2114. {
  2115. struct mem_cgroup *old = stock->cached;
  2116. if (stock->nr_pages) {
  2117. unsigned long bytes = stock->nr_pages * PAGE_SIZE;
  2118. res_counter_uncharge(&old->res, bytes);
  2119. if (do_swap_account)
  2120. res_counter_uncharge(&old->memsw, bytes);
  2121. stock->nr_pages = 0;
  2122. }
  2123. stock->cached = NULL;
  2124. }
  2125. /*
  2126. * This must be called under preempt disabled or must be called by
  2127. * a thread which is pinned to local cpu.
  2128. */
  2129. static void drain_local_stock(struct work_struct *dummy)
  2130. {
  2131. struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
  2132. drain_stock(stock);
  2133. clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
  2134. }
  2135. static void __init memcg_stock_init(void)
  2136. {
  2137. int cpu;
  2138. for_each_possible_cpu(cpu) {
  2139. struct memcg_stock_pcp *stock =
  2140. &per_cpu(memcg_stock, cpu);
  2141. INIT_WORK(&stock->work, drain_local_stock);
  2142. }
  2143. }
  2144. /*
  2145. * Cache charges(val) which is from res_counter, to local per_cpu area.
  2146. * This will be consumed by consume_stock() function, later.
  2147. */
  2148. static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
  2149. {
  2150. struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
  2151. if (stock->cached != memcg) { /* reset if necessary */
  2152. drain_stock(stock);
  2153. stock->cached = memcg;
  2154. }
  2155. stock->nr_pages += nr_pages;
  2156. put_cpu_var(memcg_stock);
  2157. }
  2158. /*
  2159. * Drains all per-CPU charge caches for given root_memcg resp. subtree
  2160. * of the hierarchy under it. sync flag says whether we should block
  2161. * until the work is done.
  2162. */
  2163. static void drain_all_stock(struct mem_cgroup *root_memcg, bool sync)
  2164. {
  2165. int cpu, curcpu;
  2166. /* Notify other cpus that system-wide "drain" is running */
  2167. get_online_cpus();
  2168. curcpu = get_cpu();
  2169. for_each_online_cpu(cpu) {
  2170. struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
  2171. struct mem_cgroup *memcg;
  2172. memcg = stock->cached;
  2173. if (!memcg || !stock->nr_pages)
  2174. continue;
  2175. if (!mem_cgroup_same_or_subtree(root_memcg, memcg))
  2176. continue;
  2177. if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
  2178. if (cpu == curcpu)
  2179. drain_local_stock(&stock->work);
  2180. else
  2181. schedule_work_on(cpu, &stock->work);
  2182. }
  2183. }
  2184. put_cpu();
  2185. if (!sync)
  2186. goto out;
  2187. for_each_online_cpu(cpu) {
  2188. struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
  2189. if (test_bit(FLUSHING_CACHED_CHARGE, &stock->flags))
  2190. flush_work(&stock->work);
  2191. }
  2192. out:
  2193. put_online_cpus();
  2194. }
  2195. /*
  2196. * Tries to drain stocked charges in other cpus. This function is asynchronous
  2197. * and just put a work per cpu for draining localy on each cpu. Caller can
  2198. * expects some charges will be back to res_counter later but cannot wait for
  2199. * it.
  2200. */
  2201. static void drain_all_stock_async(struct mem_cgroup *root_memcg)
  2202. {
  2203. /*
  2204. * If someone calls draining, avoid adding more kworker runs.
  2205. */
  2206. if (!mutex_trylock(&percpu_charge_mutex))
  2207. return;
  2208. drain_all_stock(root_memcg, false);
  2209. mutex_unlock(&percpu_charge_mutex);
  2210. }
  2211. /* This is a synchronous drain interface. */
  2212. static void drain_all_stock_sync(struct mem_cgroup *root_memcg)
  2213. {
  2214. /* called when force_empty is called */
  2215. mutex_lock(&percpu_charge_mutex);
  2216. drain_all_stock(root_memcg, true);
  2217. mutex_unlock(&percpu_charge_mutex);
  2218. }
  2219. /*
  2220. * This function drains percpu counter value from DEAD cpu and
  2221. * move it to local cpu. Note that this function can be preempted.
  2222. */
  2223. static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *memcg, int cpu)
  2224. {
  2225. int i;
  2226. spin_lock(&memcg->pcp_counter_lock);
  2227. for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
  2228. long x = per_cpu(memcg->stat->count[i], cpu);
  2229. per_cpu(memcg->stat->count[i], cpu) = 0;
  2230. memcg->nocpu_base.count[i] += x;
  2231. }
  2232. for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
  2233. unsigned long x = per_cpu(memcg->stat->events[i], cpu);
  2234. per_cpu(memcg->stat->events[i], cpu) = 0;
  2235. memcg->nocpu_base.events[i] += x;
  2236. }
  2237. spin_unlock(&memcg->pcp_counter_lock);
  2238. }
  2239. static int memcg_cpu_hotplug_callback(struct notifier_block *nb,
  2240. unsigned long action,
  2241. void *hcpu)
  2242. {
  2243. int cpu = (unsigned long)hcpu;
  2244. struct memcg_stock_pcp *stock;
  2245. struct mem_cgroup *iter;
  2246. if (action == CPU_ONLINE)
  2247. return NOTIFY_OK;
  2248. if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
  2249. return NOTIFY_OK;
  2250. for_each_mem_cgroup(iter)
  2251. mem_cgroup_drain_pcp_counter(iter, cpu);
  2252. stock = &per_cpu(memcg_stock, cpu);
  2253. drain_stock(stock);
  2254. return NOTIFY_OK;
  2255. }
  2256. /* See __mem_cgroup_try_charge() for details */
  2257. enum {
  2258. CHARGE_OK, /* success */
  2259. CHARGE_RETRY, /* need to retry but retry is not bad */
  2260. CHARGE_NOMEM, /* we can't do more. return -ENOMEM */
  2261. CHARGE_WOULDBLOCK, /* GFP_WAIT wasn't set and no enough res. */
  2262. };
  2263. static int mem_cgroup_do_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
  2264. unsigned int nr_pages, unsigned int min_pages,
  2265. bool invoke_oom)
  2266. {
  2267. unsigned long csize = nr_pages * PAGE_SIZE;
  2268. struct mem_cgroup *mem_over_limit;
  2269. struct res_counter *fail_res;
  2270. unsigned long flags = 0;
  2271. int ret;
  2272. ret = res_counter_charge(&memcg->res, csize, &fail_res);
  2273. if (likely(!ret)) {
  2274. if (!do_swap_account)
  2275. return CHARGE_OK;
  2276. ret = res_counter_charge(&memcg->memsw, csize, &fail_res);
  2277. if (likely(!ret))
  2278. return CHARGE_OK;
  2279. res_counter_uncharge(&memcg->res, csize);
  2280. mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw);
  2281. flags |= MEM_CGROUP_RECLAIM_NOSWAP;
  2282. } else
  2283. mem_over_limit = mem_cgroup_from_res_counter(fail_res, res);
  2284. /*
  2285. * Never reclaim on behalf of optional batching, retry with a
  2286. * single page instead.
  2287. */
  2288. if (nr_pages > min_pages)
  2289. return CHARGE_RETRY;
  2290. if (!(gfp_mask & __GFP_WAIT))
  2291. return CHARGE_WOULDBLOCK;
  2292. if (gfp_mask & __GFP_NORETRY)
  2293. return CHARGE_NOMEM;
  2294. ret = mem_cgroup_reclaim(mem_over_limit, gfp_mask, flags);
  2295. if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
  2296. return CHARGE_RETRY;
  2297. /*
  2298. * Even though the limit is exceeded at this point, reclaim
  2299. * may have been able to free some pages. Retry the charge
  2300. * before killing the task.
  2301. *
  2302. * Only for regular pages, though: huge pages are rather
  2303. * unlikely to succeed so close to the limit, and we fall back
  2304. * to regular pages anyway in case of failure.
  2305. */
  2306. if (nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER) && ret)
  2307. return CHARGE_RETRY;
  2308. /*
  2309. * At task move, charge accounts can be doubly counted. So, it's
  2310. * better to wait until the end of task_move if something is going on.
  2311. */
  2312. if (mem_cgroup_wait_acct_move(mem_over_limit))
  2313. return CHARGE_RETRY;
  2314. if (invoke_oom)
  2315. mem_cgroup_oom(mem_over_limit, gfp_mask, get_order(csize));
  2316. return CHARGE_NOMEM;
  2317. }
  2318. /*
  2319. * __mem_cgroup_try_charge() does
  2320. * 1. detect memcg to be charged against from passed *mm and *ptr,
  2321. * 2. update res_counter
  2322. * 3. call memory reclaim if necessary.
  2323. *
  2324. * In some special case, if the task is fatal, fatal_signal_pending() or
  2325. * has TIF_MEMDIE, this function returns -EINTR while writing root_mem_cgroup
  2326. * to *ptr. There are two reasons for this. 1: fatal threads should quit as soon
  2327. * as possible without any hazards. 2: all pages should have a valid
  2328. * pc->mem_cgroup. If mm is NULL and the caller doesn't pass a valid memcg
  2329. * pointer, that is treated as a charge to root_mem_cgroup.
  2330. *
  2331. * So __mem_cgroup_try_charge() will return
  2332. * 0 ... on success, filling *ptr with a valid memcg pointer.
  2333. * -ENOMEM ... charge failure because of resource limits.
  2334. * -EINTR ... if thread is fatal. *ptr is filled with root_mem_cgroup.
  2335. *
  2336. * Unlike the exported interface, an "oom" parameter is added. if oom==true,
  2337. * the oom-killer can be invoked.
  2338. */
  2339. static int __mem_cgroup_try_charge(struct mm_struct *mm,
  2340. gfp_t gfp_mask,
  2341. unsigned int nr_pages,
  2342. struct mem_cgroup **ptr,
  2343. bool oom)
  2344. {
  2345. unsigned int batch = max(CHARGE_BATCH, nr_pages);
  2346. int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
  2347. struct mem_cgroup *memcg = NULL;
  2348. int ret;
  2349. /*
  2350. * Unlike gloval-vm's OOM-kill, we're not in memory shortage
  2351. * in system level. So, allow to go ahead dying process in addition to
  2352. * MEMDIE process.
  2353. */
  2354. if (unlikely(test_thread_flag(TIF_MEMDIE)
  2355. || fatal_signal_pending(current)))
  2356. goto bypass;
  2357. if (unlikely(task_in_memcg_oom(current)))
  2358. goto bypass;
  2359. /*
  2360. * We always charge the cgroup the mm_struct belongs to.
  2361. * The mm_struct's mem_cgroup changes on task migration if the
  2362. * thread group leader migrates. It's possible that mm is not
  2363. * set, if so charge the root memcg (happens for pagecache usage).
  2364. */
  2365. if (!*ptr && !mm)
  2366. *ptr = root_mem_cgroup;
  2367. again:
  2368. if (*ptr) { /* css should be a valid one */
  2369. memcg = *ptr;
  2370. if (mem_cgroup_is_root(memcg))
  2371. goto done;
  2372. if (consume_stock(memcg, nr_pages))
  2373. goto done;
  2374. css_get(&memcg->css);
  2375. } else {
  2376. struct task_struct *p;
  2377. rcu_read_lock();
  2378. p = rcu_dereference(mm->owner);
  2379. /*
  2380. * Because we don't have task_lock(), "p" can exit.
  2381. * In that case, "memcg" can point to root or p can be NULL with
  2382. * race with swapoff. Then, we have small risk of mis-accouning.
  2383. * But such kind of mis-account by race always happens because
  2384. * we don't have cgroup_mutex(). It's overkill and we allo that
  2385. * small race, here.
  2386. * (*) swapoff at el will charge against mm-struct not against
  2387. * task-struct. So, mm->owner can be NULL.
  2388. */
  2389. memcg = mem_cgroup_from_task(p);
  2390. if (!memcg)
  2391. memcg = root_mem_cgroup;
  2392. if (mem_cgroup_is_root(memcg)) {
  2393. rcu_read_unlock();
  2394. goto done;
  2395. }
  2396. if (consume_stock(memcg, nr_pages)) {
  2397. /*
  2398. * It seems dagerous to access memcg without css_get().
  2399. * But considering how consume_stok works, it's not
  2400. * necessary. If consume_stock success, some charges
  2401. * from this memcg are cached on this cpu. So, we
  2402. * don't need to call css_get()/css_tryget() before
  2403. * calling consume_stock().
  2404. */
  2405. rcu_read_unlock();
  2406. goto done;
  2407. }
  2408. /* after here, we may be blocked. we need to get refcnt */
  2409. if (!css_tryget(&memcg->css)) {
  2410. rcu_read_unlock();
  2411. goto again;
  2412. }
  2413. rcu_read_unlock();
  2414. }
  2415. do {
  2416. bool invoke_oom = oom && !nr_oom_retries;
  2417. /* If killed, bypass charge */
  2418. if (fatal_signal_pending(current)) {
  2419. css_put(&memcg->css);
  2420. goto bypass;
  2421. }
  2422. ret = mem_cgroup_do_charge(memcg, gfp_mask, batch,
  2423. nr_pages, invoke_oom);
  2424. switch (ret) {
  2425. case CHARGE_OK:
  2426. break;
  2427. case CHARGE_RETRY: /* not in OOM situation but retry */
  2428. batch = nr_pages;
  2429. css_put(&memcg->css);
  2430. memcg = NULL;
  2431. goto again;
  2432. case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */
  2433. css_put(&memcg->css);
  2434. goto nomem;
  2435. case CHARGE_NOMEM: /* OOM routine works */
  2436. if (!oom || invoke_oom) {
  2437. css_put(&memcg->css);
  2438. goto nomem;
  2439. }
  2440. nr_oom_retries--;
  2441. break;
  2442. }
  2443. } while (ret != CHARGE_OK);
  2444. if (batch > nr_pages)
  2445. refill_stock(memcg, batch - nr_pages);
  2446. css_put(&memcg->css);
  2447. done:
  2448. *ptr = memcg;
  2449. return 0;
  2450. nomem:
  2451. if (!(gfp_mask & __GFP_NOFAIL)) {
  2452. *ptr = NULL;
  2453. return -ENOMEM;
  2454. }
  2455. bypass:
  2456. *ptr = root_mem_cgroup;
  2457. return -EINTR;
  2458. }
  2459. /*
  2460. * Somemtimes we have to undo a charge we got by try_charge().
  2461. * This function is for that and do uncharge, put css's refcnt.
  2462. * gotten by try_charge().
  2463. */
  2464. static void __mem_cgroup_cancel_charge(struct mem_cgroup *memcg,
  2465. unsigned int nr_pages)
  2466. {
  2467. if (!mem_cgroup_is_root(memcg)) {
  2468. unsigned long bytes = nr_pages * PAGE_SIZE;
  2469. res_counter_uncharge(&memcg->res, bytes);
  2470. if (do_swap_account)
  2471. res_counter_uncharge(&memcg->memsw, bytes);
  2472. }
  2473. }
  2474. /*
  2475. * Cancel chrages in this cgroup....doesn't propagate to parent cgroup.
  2476. * This is useful when moving usage to parent cgroup.
  2477. */
  2478. static void __mem_cgroup_cancel_local_charge(struct mem_cgroup *memcg,
  2479. unsigned int nr_pages)
  2480. {
  2481. unsigned long bytes = nr_pages * PAGE_SIZE;
  2482. if (mem_cgroup_is_root(memcg))
  2483. return;
  2484. res_counter_uncharge_until(&memcg->res, memcg->res.parent, bytes);
  2485. if (do_swap_account)
  2486. res_counter_uncharge_until(&memcg->memsw,
  2487. memcg->memsw.parent, bytes);
  2488. }
  2489. /*
  2490. * A helper function to get mem_cgroup from ID. must be called under
  2491. * rcu_read_lock(). The caller is responsible for calling css_tryget if
  2492. * the mem_cgroup is used for charging. (dropping refcnt from swap can be
  2493. * called against removed memcg.)
  2494. */
  2495. static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
  2496. {
  2497. /* ID 0 is unused ID */
  2498. if (!id)
  2499. return NULL;
  2500. return mem_cgroup_from_id(id);
  2501. }
  2502. struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
  2503. {
  2504. struct mem_cgroup *memcg = NULL;
  2505. struct page_cgroup *pc;
  2506. unsigned short id;
  2507. swp_entry_t ent;
  2508. VM_BUG_ON(!PageLocked(page));
  2509. pc = lookup_page_cgroup(page);
  2510. lock_page_cgroup(pc);
  2511. if (PageCgroupUsed(pc)) {
  2512. memcg = pc->mem_cgroup;
  2513. if (memcg && !css_tryget(&memcg->css))
  2514. memcg = NULL;
  2515. } else if (PageSwapCache(page)) {
  2516. ent.val = page_private(page);
  2517. id = lookup_swap_cgroup_id(ent);
  2518. rcu_read_lock();
  2519. memcg = mem_cgroup_lookup(id);
  2520. if (memcg && !css_tryget(&memcg->css))
  2521. memcg = NULL;
  2522. rcu_read_unlock();
  2523. }
  2524. unlock_page_cgroup(pc);
  2525. return memcg;
  2526. }
  2527. static void __mem_cgroup_commit_charge(struct mem_cgroup *memcg,
  2528. struct page *page,
  2529. unsigned int nr_pages,
  2530. enum charge_type ctype,
  2531. bool lrucare)
  2532. {
  2533. struct page_cgroup *pc = lookup_page_cgroup(page);
  2534. struct zone *uninitialized_var(zone);
  2535. struct lruvec *lruvec;
  2536. bool was_on_lru = false;
  2537. bool anon;
  2538. lock_page_cgroup(pc);
  2539. VM_BUG_ON(PageCgroupUsed(pc));
  2540. /*
  2541. * we don't need page_cgroup_lock about tail pages, becase they are not
  2542. * accessed by any other context at this point.
  2543. */
  2544. /*
  2545. * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
  2546. * may already be on some other mem_cgroup's LRU. Take care of it.
  2547. */
  2548. if (lrucare) {
  2549. zone = page_zone(page);
  2550. spin_lock_irq(&zone->lru_lock);
  2551. if (PageLRU(page)) {
  2552. lruvec = mem_cgroup_zone_lruvec(zone, pc->mem_cgroup);
  2553. ClearPageLRU(page);
  2554. del_page_from_lru_list(page, lruvec, page_lru(page));
  2555. was_on_lru = true;
  2556. }
  2557. }
  2558. pc->mem_cgroup = memcg;
  2559. /*
  2560. * We access a page_cgroup asynchronously without lock_page_cgroup().
  2561. * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
  2562. * is accessed after testing USED bit. To make pc->mem_cgroup visible
  2563. * before USED bit, we need memory barrier here.
  2564. * See mem_cgroup_add_lru_list(), etc.
  2565. */
  2566. smp_wmb();
  2567. SetPageCgroupUsed(pc);
  2568. if (lrucare) {
  2569. if (was_on_lru) {
  2570. lruvec = mem_cgroup_zone_lruvec(zone, pc->mem_cgroup);
  2571. VM_BUG_ON(PageLRU(page));
  2572. SetPageLRU(page);
  2573. add_page_to_lru_list(page, lruvec, page_lru(page));
  2574. }
  2575. spin_unlock_irq(&zone->lru_lock);
  2576. }
  2577. if (ctype == MEM_CGROUP_CHARGE_TYPE_ANON)
  2578. anon = true;
  2579. else
  2580. anon = false;
  2581. mem_cgroup_charge_statistics(memcg, page, anon, nr_pages);
  2582. unlock_page_cgroup(pc);
  2583. /*
  2584. * "charge_statistics" updated event counter. Then, check it.
  2585. * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
  2586. * if they exceeds softlimit.
  2587. */
  2588. memcg_check_events(memcg, page);
  2589. }
  2590. static DEFINE_MUTEX(set_limit_mutex);
  2591. #ifdef CONFIG_MEMCG_KMEM
  2592. static inline bool memcg_can_account_kmem(struct mem_cgroup *memcg)
  2593. {
  2594. return !mem_cgroup_disabled() && !mem_cgroup_is_root(memcg) &&
  2595. (memcg->kmem_account_flags & KMEM_ACCOUNTED_MASK);
  2596. }
  2597. /*
  2598. * This is a bit cumbersome, but it is rarely used and avoids a backpointer
  2599. * in the memcg_cache_params struct.
  2600. */
  2601. static struct kmem_cache *memcg_params_to_cache(struct memcg_cache_params *p)
  2602. {
  2603. struct kmem_cache *cachep;
  2604. VM_BUG_ON(p->is_root_cache);
  2605. cachep = p->root_cache;
  2606. return cache_from_memcg_idx(cachep, memcg_cache_id(p->memcg));
  2607. }
  2608. #ifdef CONFIG_SLABINFO
  2609. static int mem_cgroup_slabinfo_read(struct cgroup_subsys_state *css,
  2610. struct cftype *cft, struct seq_file *m)
  2611. {
  2612. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  2613. struct memcg_cache_params *params;
  2614. if (!memcg_can_account_kmem(memcg))
  2615. return -EIO;
  2616. print_slabinfo_header(m);
  2617. mutex_lock(&memcg->slab_caches_mutex);
  2618. list_for_each_entry(params, &memcg->memcg_slab_caches, list)
  2619. cache_show(memcg_params_to_cache(params), m);
  2620. mutex_unlock(&memcg->slab_caches_mutex);
  2621. return 0;
  2622. }
  2623. #endif
  2624. static int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp, u64 size)
  2625. {
  2626. struct res_counter *fail_res;
  2627. struct mem_cgroup *_memcg;
  2628. int ret = 0;
  2629. ret = res_counter_charge(&memcg->kmem, size, &fail_res);
  2630. if (ret)
  2631. return ret;
  2632. _memcg = memcg;
  2633. ret = __mem_cgroup_try_charge(NULL, gfp, size >> PAGE_SHIFT,
  2634. &_memcg, oom_gfp_allowed(gfp));
  2635. if (ret == -EINTR) {
  2636. /*
  2637. * __mem_cgroup_try_charge() chosed to bypass to root due to
  2638. * OOM kill or fatal signal. Since our only options are to
  2639. * either fail the allocation or charge it to this cgroup, do
  2640. * it as a temporary condition. But we can't fail. From a
  2641. * kmem/slab perspective, the cache has already been selected,
  2642. * by mem_cgroup_kmem_get_cache(), so it is too late to change
  2643. * our minds.
  2644. *
  2645. * This condition will only trigger if the task entered
  2646. * memcg_charge_kmem in a sane state, but was OOM-killed during
  2647. * __mem_cgroup_try_charge() above. Tasks that were already
  2648. * dying when the allocation triggers should have been already
  2649. * directed to the root cgroup in memcontrol.h
  2650. */
  2651. res_counter_charge_nofail(&memcg->res, size, &fail_res);
  2652. if (do_swap_account)
  2653. res_counter_charge_nofail(&memcg->memsw, size,
  2654. &fail_res);
  2655. ret = 0;
  2656. } else if (ret)
  2657. res_counter_uncharge(&memcg->kmem, size);
  2658. return ret;
  2659. }
  2660. static void memcg_uncharge_kmem(struct mem_cgroup *memcg, u64 size)
  2661. {
  2662. res_counter_uncharge(&memcg->res, size);
  2663. if (do_swap_account)
  2664. res_counter_uncharge(&memcg->memsw, size);
  2665. /* Not down to 0 */
  2666. if (res_counter_uncharge(&memcg->kmem, size))
  2667. return;
  2668. /*
  2669. * Releases a reference taken in kmem_cgroup_css_offline in case
  2670. * this last uncharge is racing with the offlining code or it is
  2671. * outliving the memcg existence.
  2672. *
  2673. * The memory barrier imposed by test&clear is paired with the
  2674. * explicit one in memcg_kmem_mark_dead().
  2675. */
  2676. if (memcg_kmem_test_and_clear_dead(memcg))
  2677. css_put(&memcg->css);
  2678. }
  2679. void memcg_cache_list_add(struct mem_cgroup *memcg, struct kmem_cache *cachep)
  2680. {
  2681. if (!memcg)
  2682. return;
  2683. mutex_lock(&memcg->slab_caches_mutex);
  2684. list_add(&cachep->memcg_params->list, &memcg->memcg_slab_caches);
  2685. mutex_unlock(&memcg->slab_caches_mutex);
  2686. }
  2687. /*
  2688. * helper for acessing a memcg's index. It will be used as an index in the
  2689. * child cache array in kmem_cache, and also to derive its name. This function
  2690. * will return -1 when this is not a kmem-limited memcg.
  2691. */
  2692. int memcg_cache_id(struct mem_cgroup *memcg)
  2693. {
  2694. return memcg ? memcg->kmemcg_id : -1;
  2695. }
  2696. /*
  2697. * This ends up being protected by the set_limit mutex, during normal
  2698. * operation, because that is its main call site.
  2699. *
  2700. * But when we create a new cache, we can call this as well if its parent
  2701. * is kmem-limited. That will have to hold set_limit_mutex as well.
  2702. */
  2703. int memcg_update_cache_sizes(struct mem_cgroup *memcg)
  2704. {
  2705. int num, ret;
  2706. num = ida_simple_get(&kmem_limited_groups,
  2707. 0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
  2708. if (num < 0)
  2709. return num;
  2710. /*
  2711. * After this point, kmem_accounted (that we test atomically in
  2712. * the beginning of this conditional), is no longer 0. This
  2713. * guarantees only one process will set the following boolean
  2714. * to true. We don't need test_and_set because we're protected
  2715. * by the set_limit_mutex anyway.
  2716. */
  2717. memcg_kmem_set_activated(memcg);
  2718. ret = memcg_update_all_caches(num+1);
  2719. if (ret) {
  2720. ida_simple_remove(&kmem_limited_groups, num);
  2721. memcg_kmem_clear_activated(memcg);
  2722. return ret;
  2723. }
  2724. memcg->kmemcg_id = num;
  2725. INIT_LIST_HEAD(&memcg->memcg_slab_caches);
  2726. mutex_init(&memcg->slab_caches_mutex);
  2727. return 0;
  2728. }
  2729. static size_t memcg_caches_array_size(int num_groups)
  2730. {
  2731. ssize_t size;
  2732. if (num_groups <= 0)
  2733. return 0;
  2734. size = 2 * num_groups;
  2735. if (size < MEMCG_CACHES_MIN_SIZE)
  2736. size = MEMCG_CACHES_MIN_SIZE;
  2737. else if (size > MEMCG_CACHES_MAX_SIZE)
  2738. size = MEMCG_CACHES_MAX_SIZE;
  2739. return size;
  2740. }
  2741. /*
  2742. * We should update the current array size iff all caches updates succeed. This
  2743. * can only be done from the slab side. The slab mutex needs to be held when
  2744. * calling this.
  2745. */
  2746. void memcg_update_array_size(int num)
  2747. {
  2748. if (num > memcg_limited_groups_array_size)
  2749. memcg_limited_groups_array_size = memcg_caches_array_size(num);
  2750. }
  2751. static void kmem_cache_destroy_work_func(struct work_struct *w);
  2752. int memcg_update_cache_size(struct kmem_cache *s, int num_groups)
  2753. {
  2754. struct memcg_cache_params *cur_params = s->memcg_params;
  2755. VM_BUG_ON(!is_root_cache(s));
  2756. if (num_groups > memcg_limited_groups_array_size) {
  2757. int i;
  2758. ssize_t size = memcg_caches_array_size(num_groups);
  2759. size *= sizeof(void *);
  2760. size += offsetof(struct memcg_cache_params, memcg_caches);
  2761. s->memcg_params = kzalloc(size, GFP_KERNEL);
  2762. if (!s->memcg_params) {
  2763. s->memcg_params = cur_params;
  2764. return -ENOMEM;
  2765. }
  2766. s->memcg_params->is_root_cache = true;
  2767. /*
  2768. * There is the chance it will be bigger than
  2769. * memcg_limited_groups_array_size, if we failed an allocation
  2770. * in a cache, in which case all caches updated before it, will
  2771. * have a bigger array.
  2772. *
  2773. * But if that is the case, the data after
  2774. * memcg_limited_groups_array_size is certainly unused
  2775. */
  2776. for (i = 0; i < memcg_limited_groups_array_size; i++) {
  2777. if (!cur_params->memcg_caches[i])
  2778. continue;
  2779. s->memcg_params->memcg_caches[i] =
  2780. cur_params->memcg_caches[i];
  2781. }
  2782. /*
  2783. * Ideally, we would wait until all caches succeed, and only
  2784. * then free the old one. But this is not worth the extra
  2785. * pointer per-cache we'd have to have for this.
  2786. *
  2787. * It is not a big deal if some caches are left with a size
  2788. * bigger than the others. And all updates will reset this
  2789. * anyway.
  2790. */
  2791. kfree(cur_params);
  2792. }
  2793. return 0;
  2794. }
  2795. int memcg_register_cache(struct mem_cgroup *memcg, struct kmem_cache *s,
  2796. struct kmem_cache *root_cache)
  2797. {
  2798. size_t size;
  2799. if (!memcg_kmem_enabled())
  2800. return 0;
  2801. if (!memcg) {
  2802. size = offsetof(struct memcg_cache_params, memcg_caches);
  2803. size += memcg_limited_groups_array_size * sizeof(void *);
  2804. } else
  2805. size = sizeof(struct memcg_cache_params);
  2806. s->memcg_params = kzalloc(size, GFP_KERNEL);
  2807. if (!s->memcg_params)
  2808. return -ENOMEM;
  2809. if (memcg) {
  2810. s->memcg_params->memcg = memcg;
  2811. s->memcg_params->root_cache = root_cache;
  2812. INIT_WORK(&s->memcg_params->destroy,
  2813. kmem_cache_destroy_work_func);
  2814. } else
  2815. s->memcg_params->is_root_cache = true;
  2816. return 0;
  2817. }
  2818. void memcg_release_cache(struct kmem_cache *s)
  2819. {
  2820. struct kmem_cache *root;
  2821. struct mem_cgroup *memcg;
  2822. int id;
  2823. /*
  2824. * This happens, for instance, when a root cache goes away before we
  2825. * add any memcg.
  2826. */
  2827. if (!s->memcg_params)
  2828. return;
  2829. if (s->memcg_params->is_root_cache)
  2830. goto out;
  2831. memcg = s->memcg_params->memcg;
  2832. id = memcg_cache_id(memcg);
  2833. root = s->memcg_params->root_cache;
  2834. root->memcg_params->memcg_caches[id] = NULL;
  2835. mutex_lock(&memcg->slab_caches_mutex);
  2836. list_del(&s->memcg_params->list);
  2837. mutex_unlock(&memcg->slab_caches_mutex);
  2838. css_put(&memcg->css);
  2839. out:
  2840. kfree(s->memcg_params);
  2841. }
  2842. /*
  2843. * During the creation a new cache, we need to disable our accounting mechanism
  2844. * altogether. This is true even if we are not creating, but rather just
  2845. * enqueing new caches to be created.
  2846. *
  2847. * This is because that process will trigger allocations; some visible, like
  2848. * explicit kmallocs to auxiliary data structures, name strings and internal
  2849. * cache structures; some well concealed, like INIT_WORK() that can allocate
  2850. * objects during debug.
  2851. *
  2852. * If any allocation happens during memcg_kmem_get_cache, we will recurse back
  2853. * to it. This may not be a bounded recursion: since the first cache creation
  2854. * failed to complete (waiting on the allocation), we'll just try to create the
  2855. * cache again, failing at the same point.
  2856. *
  2857. * memcg_kmem_get_cache is prepared to abort after seeing a positive count of
  2858. * memcg_kmem_skip_account. So we enclose anything that might allocate memory
  2859. * inside the following two functions.
  2860. */
  2861. static inline void memcg_stop_kmem_account(void)
  2862. {
  2863. VM_BUG_ON(!current->mm);
  2864. current->memcg_kmem_skip_account++;
  2865. }
  2866. static inline void memcg_resume_kmem_account(void)
  2867. {
  2868. VM_BUG_ON(!current->mm);
  2869. current->memcg_kmem_skip_account--;
  2870. }
  2871. static void kmem_cache_destroy_work_func(struct work_struct *w)
  2872. {
  2873. struct kmem_cache *cachep;
  2874. struct memcg_cache_params *p;
  2875. p = container_of(w, struct memcg_cache_params, destroy);
  2876. cachep = memcg_params_to_cache(p);
  2877. /*
  2878. * If we get down to 0 after shrink, we could delete right away.
  2879. * However, memcg_release_pages() already puts us back in the workqueue
  2880. * in that case. If we proceed deleting, we'll get a dangling
  2881. * reference, and removing the object from the workqueue in that case
  2882. * is unnecessary complication. We are not a fast path.
  2883. *
  2884. * Note that this case is fundamentally different from racing with
  2885. * shrink_slab(): if memcg_cgroup_destroy_cache() is called in
  2886. * kmem_cache_shrink, not only we would be reinserting a dead cache
  2887. * into the queue, but doing so from inside the worker racing to
  2888. * destroy it.
  2889. *
  2890. * So if we aren't down to zero, we'll just schedule a worker and try
  2891. * again
  2892. */
  2893. if (atomic_read(&cachep->memcg_params->nr_pages) != 0) {
  2894. kmem_cache_shrink(cachep);
  2895. if (atomic_read(&cachep->memcg_params->nr_pages) == 0)
  2896. return;
  2897. } else
  2898. kmem_cache_destroy(cachep);
  2899. }
  2900. void mem_cgroup_destroy_cache(struct kmem_cache *cachep)
  2901. {
  2902. if (!cachep->memcg_params->dead)
  2903. return;
  2904. /*
  2905. * There are many ways in which we can get here.
  2906. *
  2907. * We can get to a memory-pressure situation while the delayed work is
  2908. * still pending to run. The vmscan shrinkers can then release all
  2909. * cache memory and get us to destruction. If this is the case, we'll
  2910. * be executed twice, which is a bug (the second time will execute over
  2911. * bogus data). In this case, cancelling the work should be fine.
  2912. *
  2913. * But we can also get here from the worker itself, if
  2914. * kmem_cache_shrink is enough to shake all the remaining objects and
  2915. * get the page count to 0. In this case, we'll deadlock if we try to
  2916. * cancel the work (the worker runs with an internal lock held, which
  2917. * is the same lock we would hold for cancel_work_sync().)
  2918. *
  2919. * Since we can't possibly know who got us here, just refrain from
  2920. * running if there is already work pending
  2921. */
  2922. if (work_pending(&cachep->memcg_params->destroy))
  2923. return;
  2924. /*
  2925. * We have to defer the actual destroying to a workqueue, because
  2926. * we might currently be in a context that cannot sleep.
  2927. */
  2928. schedule_work(&cachep->memcg_params->destroy);
  2929. }
  2930. /*
  2931. * This lock protects updaters, not readers. We want readers to be as fast as
  2932. * they can, and they will either see NULL or a valid cache value. Our model
  2933. * allow them to see NULL, in which case the root memcg will be selected.
  2934. *
  2935. * We need this lock because multiple allocations to the same cache from a non
  2936. * will span more than one worker. Only one of them can create the cache.
  2937. */
  2938. static DEFINE_MUTEX(memcg_cache_mutex);
  2939. /*
  2940. * Called with memcg_cache_mutex held
  2941. */
  2942. static struct kmem_cache *kmem_cache_dup(struct mem_cgroup *memcg,
  2943. struct kmem_cache *s)
  2944. {
  2945. struct kmem_cache *new;
  2946. static char *tmp_name = NULL;
  2947. lockdep_assert_held(&memcg_cache_mutex);
  2948. /*
  2949. * kmem_cache_create_memcg duplicates the given name and
  2950. * cgroup_name for this name requires RCU context.
  2951. * This static temporary buffer is used to prevent from
  2952. * pointless shortliving allocation.
  2953. */
  2954. if (!tmp_name) {
  2955. tmp_name = kmalloc(PATH_MAX, GFP_KERNEL);
  2956. if (!tmp_name)
  2957. return NULL;
  2958. }
  2959. rcu_read_lock();
  2960. snprintf(tmp_name, PATH_MAX, "%s(%d:%s)", s->name,
  2961. memcg_cache_id(memcg), cgroup_name(memcg->css.cgroup));
  2962. rcu_read_unlock();
  2963. new = kmem_cache_create_memcg(memcg, tmp_name, s->object_size, s->align,
  2964. (s->flags & ~SLAB_PANIC), s->ctor, s);
  2965. if (new)
  2966. new->allocflags |= __GFP_KMEMCG;
  2967. return new;
  2968. }
  2969. static struct kmem_cache *memcg_create_kmem_cache(struct mem_cgroup *memcg,
  2970. struct kmem_cache *cachep)
  2971. {
  2972. struct kmem_cache *new_cachep;
  2973. int idx;
  2974. BUG_ON(!memcg_can_account_kmem(memcg));
  2975. idx = memcg_cache_id(memcg);
  2976. mutex_lock(&memcg_cache_mutex);
  2977. new_cachep = cache_from_memcg_idx(cachep, idx);
  2978. if (new_cachep) {
  2979. css_put(&memcg->css);
  2980. goto out;
  2981. }
  2982. new_cachep = kmem_cache_dup(memcg, cachep);
  2983. if (new_cachep == NULL) {
  2984. new_cachep = cachep;
  2985. css_put(&memcg->css);
  2986. goto out;
  2987. }
  2988. atomic_set(&new_cachep->memcg_params->nr_pages , 0);
  2989. cachep->memcg_params->memcg_caches[idx] = new_cachep;
  2990. /*
  2991. * the readers won't lock, make sure everybody sees the updated value,
  2992. * so they won't put stuff in the queue again for no reason
  2993. */
  2994. wmb();
  2995. out:
  2996. mutex_unlock(&memcg_cache_mutex);
  2997. return new_cachep;
  2998. }
  2999. void kmem_cache_destroy_memcg_children(struct kmem_cache *s)
  3000. {
  3001. struct kmem_cache *c;
  3002. int i;
  3003. if (!s->memcg_params)
  3004. return;
  3005. if (!s->memcg_params->is_root_cache)
  3006. return;
  3007. /*
  3008. * If the cache is being destroyed, we trust that there is no one else
  3009. * requesting objects from it. Even if there are, the sanity checks in
  3010. * kmem_cache_destroy should caught this ill-case.
  3011. *
  3012. * Still, we don't want anyone else freeing memcg_caches under our
  3013. * noses, which can happen if a new memcg comes to life. As usual,
  3014. * we'll take the set_limit_mutex to protect ourselves against this.
  3015. */
  3016. mutex_lock(&set_limit_mutex);
  3017. for_each_memcg_cache_index(i) {
  3018. c = cache_from_memcg_idx(s, i);
  3019. if (!c)
  3020. continue;
  3021. /*
  3022. * We will now manually delete the caches, so to avoid races
  3023. * we need to cancel all pending destruction workers and
  3024. * proceed with destruction ourselves.
  3025. *
  3026. * kmem_cache_destroy() will call kmem_cache_shrink internally,
  3027. * and that could spawn the workers again: it is likely that
  3028. * the cache still have active pages until this very moment.
  3029. * This would lead us back to mem_cgroup_destroy_cache.
  3030. *
  3031. * But that will not execute at all if the "dead" flag is not
  3032. * set, so flip it down to guarantee we are in control.
  3033. */
  3034. c->memcg_params->dead = false;
  3035. cancel_work_sync(&c->memcg_params->destroy);
  3036. kmem_cache_destroy(c);
  3037. }
  3038. mutex_unlock(&set_limit_mutex);
  3039. }
  3040. struct create_work {
  3041. struct mem_cgroup *memcg;
  3042. struct kmem_cache *cachep;
  3043. struct work_struct work;
  3044. };
  3045. static void mem_cgroup_destroy_all_caches(struct mem_cgroup *memcg)
  3046. {
  3047. struct kmem_cache *cachep;
  3048. struct memcg_cache_params *params;
  3049. if (!memcg_kmem_is_active(memcg))
  3050. return;
  3051. mutex_lock(&memcg->slab_caches_mutex);
  3052. list_for_each_entry(params, &memcg->memcg_slab_caches, list) {
  3053. cachep = memcg_params_to_cache(params);
  3054. cachep->memcg_params->dead = true;
  3055. schedule_work(&cachep->memcg_params->destroy);
  3056. }
  3057. mutex_unlock(&memcg->slab_caches_mutex);
  3058. }
  3059. static void memcg_create_cache_work_func(struct work_struct *w)
  3060. {
  3061. struct create_work *cw;
  3062. cw = container_of(w, struct create_work, work);
  3063. memcg_create_kmem_cache(cw->memcg, cw->cachep);
  3064. kfree(cw);
  3065. }
  3066. /*
  3067. * Enqueue the creation of a per-memcg kmem_cache.
  3068. */
  3069. static void __memcg_create_cache_enqueue(struct mem_cgroup *memcg,
  3070. struct kmem_cache *cachep)
  3071. {
  3072. struct create_work *cw;
  3073. cw = kmalloc(sizeof(struct create_work), GFP_NOWAIT);
  3074. if (cw == NULL) {
  3075. css_put(&memcg->css);
  3076. return;
  3077. }
  3078. cw->memcg = memcg;
  3079. cw->cachep = cachep;
  3080. INIT_WORK(&cw->work, memcg_create_cache_work_func);
  3081. schedule_work(&cw->work);
  3082. }
  3083. static void memcg_create_cache_enqueue(struct mem_cgroup *memcg,
  3084. struct kmem_cache *cachep)
  3085. {
  3086. /*
  3087. * We need to stop accounting when we kmalloc, because if the
  3088. * corresponding kmalloc cache is not yet created, the first allocation
  3089. * in __memcg_create_cache_enqueue will recurse.
  3090. *
  3091. * However, it is better to enclose the whole function. Depending on
  3092. * the debugging options enabled, INIT_WORK(), for instance, can
  3093. * trigger an allocation. This too, will make us recurse. Because at
  3094. * this point we can't allow ourselves back into memcg_kmem_get_cache,
  3095. * the safest choice is to do it like this, wrapping the whole function.
  3096. */
  3097. memcg_stop_kmem_account();
  3098. __memcg_create_cache_enqueue(memcg, cachep);
  3099. memcg_resume_kmem_account();
  3100. }
  3101. /*
  3102. * Return the kmem_cache we're supposed to use for a slab allocation.
  3103. * We try to use the current memcg's version of the cache.
  3104. *
  3105. * If the cache does not exist yet, if we are the first user of it,
  3106. * we either create it immediately, if possible, or create it asynchronously
  3107. * in a workqueue.
  3108. * In the latter case, we will let the current allocation go through with
  3109. * the original cache.
  3110. *
  3111. * Can't be called in interrupt context or from kernel threads.
  3112. * This function needs to be called with rcu_read_lock() held.
  3113. */
  3114. struct kmem_cache *__memcg_kmem_get_cache(struct kmem_cache *cachep,
  3115. gfp_t gfp)
  3116. {
  3117. struct mem_cgroup *memcg;
  3118. int idx;
  3119. VM_BUG_ON(!cachep->memcg_params);
  3120. VM_BUG_ON(!cachep->memcg_params->is_root_cache);
  3121. if (!current->mm || current->memcg_kmem_skip_account)
  3122. return cachep;
  3123. rcu_read_lock();
  3124. memcg = mem_cgroup_from_task(rcu_dereference(current->mm->owner));
  3125. if (!memcg_can_account_kmem(memcg))
  3126. goto out;
  3127. idx = memcg_cache_id(memcg);
  3128. /*
  3129. * barrier to mare sure we're always seeing the up to date value. The
  3130. * code updating memcg_caches will issue a write barrier to match this.
  3131. */
  3132. read_barrier_depends();
  3133. if (likely(cache_from_memcg_idx(cachep, idx))) {
  3134. cachep = cache_from_memcg_idx(cachep, idx);
  3135. goto out;
  3136. }
  3137. /* The corresponding put will be done in the workqueue. */
  3138. if (!css_tryget(&memcg->css))
  3139. goto out;
  3140. rcu_read_unlock();
  3141. /*
  3142. * If we are in a safe context (can wait, and not in interrupt
  3143. * context), we could be be predictable and return right away.
  3144. * This would guarantee that the allocation being performed
  3145. * already belongs in the new cache.
  3146. *
  3147. * However, there are some clashes that can arrive from locking.
  3148. * For instance, because we acquire the slab_mutex while doing
  3149. * kmem_cache_dup, this means no further allocation could happen
  3150. * with the slab_mutex held.
  3151. *
  3152. * Also, because cache creation issue get_online_cpus(), this
  3153. * creates a lock chain: memcg_slab_mutex -> cpu_hotplug_mutex,
  3154. * that ends up reversed during cpu hotplug. (cpuset allocates
  3155. * a bunch of GFP_KERNEL memory during cpuup). Due to all that,
  3156. * better to defer everything.
  3157. */
  3158. memcg_create_cache_enqueue(memcg, cachep);
  3159. return cachep;
  3160. out:
  3161. rcu_read_unlock();
  3162. return cachep;
  3163. }
  3164. EXPORT_SYMBOL(__memcg_kmem_get_cache);
  3165. /*
  3166. * We need to verify if the allocation against current->mm->owner's memcg is
  3167. * possible for the given order. But the page is not allocated yet, so we'll
  3168. * need a further commit step to do the final arrangements.
  3169. *
  3170. * It is possible for the task to switch cgroups in this mean time, so at
  3171. * commit time, we can't rely on task conversion any longer. We'll then use
  3172. * the handle argument to return to the caller which cgroup we should commit
  3173. * against. We could also return the memcg directly and avoid the pointer
  3174. * passing, but a boolean return value gives better semantics considering
  3175. * the compiled-out case as well.
  3176. *
  3177. * Returning true means the allocation is possible.
  3178. */
  3179. bool
  3180. __memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **_memcg, int order)
  3181. {
  3182. struct mem_cgroup *memcg;
  3183. int ret;
  3184. *_memcg = NULL;
  3185. /*
  3186. * Disabling accounting is only relevant for some specific memcg
  3187. * internal allocations. Therefore we would initially not have such
  3188. * check here, since direct calls to the page allocator that are marked
  3189. * with GFP_KMEMCG only happen outside memcg core. We are mostly
  3190. * concerned with cache allocations, and by having this test at
  3191. * memcg_kmem_get_cache, we are already able to relay the allocation to
  3192. * the root cache and bypass the memcg cache altogether.
  3193. *
  3194. * There is one exception, though: the SLUB allocator does not create
  3195. * large order caches, but rather service large kmallocs directly from
  3196. * the page allocator. Therefore, the following sequence when backed by
  3197. * the SLUB allocator:
  3198. *
  3199. * memcg_stop_kmem_account();
  3200. * kmalloc(<large_number>)
  3201. * memcg_resume_kmem_account();
  3202. *
  3203. * would effectively ignore the fact that we should skip accounting,
  3204. * since it will drive us directly to this function without passing
  3205. * through the cache selector memcg_kmem_get_cache. Such large
  3206. * allocations are extremely rare but can happen, for instance, for the
  3207. * cache arrays. We bring this test here.
  3208. */
  3209. if (!current->mm || current->memcg_kmem_skip_account)
  3210. return true;
  3211. memcg = try_get_mem_cgroup_from_mm(current->mm);
  3212. /*
  3213. * very rare case described in mem_cgroup_from_task. Unfortunately there
  3214. * isn't much we can do without complicating this too much, and it would
  3215. * be gfp-dependent anyway. Just let it go
  3216. */
  3217. if (unlikely(!memcg))
  3218. return true;
  3219. if (!memcg_can_account_kmem(memcg)) {
  3220. css_put(&memcg->css);
  3221. return true;
  3222. }
  3223. ret = memcg_charge_kmem(memcg, gfp, PAGE_SIZE << order);
  3224. if (!ret)
  3225. *_memcg = memcg;
  3226. css_put(&memcg->css);
  3227. return (ret == 0);
  3228. }
  3229. void __memcg_kmem_commit_charge(struct page *page, struct mem_cgroup *memcg,
  3230. int order)
  3231. {
  3232. struct page_cgroup *pc;
  3233. VM_BUG_ON(mem_cgroup_is_root(memcg));
  3234. /* The page allocation failed. Revert */
  3235. if (!page) {
  3236. memcg_uncharge_kmem(memcg, PAGE_SIZE << order);
  3237. return;
  3238. }
  3239. pc = lookup_page_cgroup(page);
  3240. lock_page_cgroup(pc);
  3241. pc->mem_cgroup = memcg;
  3242. SetPageCgroupUsed(pc);
  3243. unlock_page_cgroup(pc);
  3244. }
  3245. void __memcg_kmem_uncharge_pages(struct page *page, int order)
  3246. {
  3247. struct mem_cgroup *memcg = NULL;
  3248. struct page_cgroup *pc;
  3249. pc = lookup_page_cgroup(page);
  3250. /*
  3251. * Fast unlocked return. Theoretically might have changed, have to
  3252. * check again after locking.
  3253. */
  3254. if (!PageCgroupUsed(pc))
  3255. return;
  3256. lock_page_cgroup(pc);
  3257. if (PageCgroupUsed(pc)) {
  3258. memcg = pc->mem_cgroup;
  3259. ClearPageCgroupUsed(pc);
  3260. }
  3261. unlock_page_cgroup(pc);
  3262. /*
  3263. * We trust that only if there is a memcg associated with the page, it
  3264. * is a valid allocation
  3265. */
  3266. if (!memcg)
  3267. return;
  3268. VM_BUG_ON(mem_cgroup_is_root(memcg));
  3269. memcg_uncharge_kmem(memcg, PAGE_SIZE << order);
  3270. }
  3271. #else
  3272. static inline void mem_cgroup_destroy_all_caches(struct mem_cgroup *memcg)
  3273. {
  3274. }
  3275. #endif /* CONFIG_MEMCG_KMEM */
  3276. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  3277. #define PCGF_NOCOPY_AT_SPLIT (1 << PCG_LOCK | 1 << PCG_MIGRATION)
  3278. /*
  3279. * Because tail pages are not marked as "used", set it. We're under
  3280. * zone->lru_lock, 'splitting on pmd' and compound_lock.
  3281. * charge/uncharge will be never happen and move_account() is done under
  3282. * compound_lock(), so we don't have to take care of races.
  3283. */
  3284. void mem_cgroup_split_huge_fixup(struct page *head)
  3285. {
  3286. struct page_cgroup *head_pc = lookup_page_cgroup(head);
  3287. struct page_cgroup *pc;
  3288. struct mem_cgroup *memcg;
  3289. int i;
  3290. if (mem_cgroup_disabled())
  3291. return;
  3292. memcg = head_pc->mem_cgroup;
  3293. for (i = 1; i < HPAGE_PMD_NR; i++) {
  3294. pc = head_pc + i;
  3295. pc->mem_cgroup = memcg;
  3296. smp_wmb();/* see __commit_charge() */
  3297. pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT;
  3298. }
  3299. __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE],
  3300. HPAGE_PMD_NR);
  3301. }
  3302. #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  3303. static inline
  3304. void mem_cgroup_move_account_page_stat(struct mem_cgroup *from,
  3305. struct mem_cgroup *to,
  3306. unsigned int nr_pages,
  3307. enum mem_cgroup_stat_index idx)
  3308. {
  3309. /* Update stat data for mem_cgroup */
  3310. preempt_disable();
  3311. __this_cpu_sub(from->stat->count[idx], nr_pages);
  3312. __this_cpu_add(to->stat->count[idx], nr_pages);
  3313. preempt_enable();
  3314. }
  3315. /**
  3316. * mem_cgroup_move_account - move account of the page
  3317. * @page: the page
  3318. * @nr_pages: number of regular pages (>1 for huge pages)
  3319. * @pc: page_cgroup of the page.
  3320. * @from: mem_cgroup which the page is moved from.
  3321. * @to: mem_cgroup which the page is moved to. @from != @to.
  3322. *
  3323. * The caller must confirm following.
  3324. * - page is not on LRU (isolate_page() is useful.)
  3325. * - compound_lock is held when nr_pages > 1
  3326. *
  3327. * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
  3328. * from old cgroup.
  3329. */
  3330. static int mem_cgroup_move_account(struct page *page,
  3331. unsigned int nr_pages,
  3332. struct page_cgroup *pc,
  3333. struct mem_cgroup *from,
  3334. struct mem_cgroup *to)
  3335. {
  3336. unsigned long flags;
  3337. int ret;
  3338. bool anon = PageAnon(page);
  3339. VM_BUG_ON(from == to);
  3340. VM_BUG_ON(PageLRU(page));
  3341. /*
  3342. * The page is isolated from LRU. So, collapse function
  3343. * will not handle this page. But page splitting can happen.
  3344. * Do this check under compound_page_lock(). The caller should
  3345. * hold it.
  3346. */
  3347. ret = -EBUSY;
  3348. if (nr_pages > 1 && !PageTransHuge(page))
  3349. goto out;
  3350. lock_page_cgroup(pc);
  3351. ret = -EINVAL;
  3352. if (!PageCgroupUsed(pc) || pc->mem_cgroup != from)
  3353. goto unlock;
  3354. move_lock_mem_cgroup(from, &flags);
  3355. if (!anon && page_mapped(page))
  3356. mem_cgroup_move_account_page_stat(from, to, nr_pages,
  3357. MEM_CGROUP_STAT_FILE_MAPPED);
  3358. if (PageWriteback(page))
  3359. mem_cgroup_move_account_page_stat(from, to, nr_pages,
  3360. MEM_CGROUP_STAT_WRITEBACK);
  3361. mem_cgroup_charge_statistics(from, page, anon, -nr_pages);
  3362. /* caller should have done css_get */
  3363. pc->mem_cgroup = to;
  3364. mem_cgroup_charge_statistics(to, page, anon, nr_pages);
  3365. move_unlock_mem_cgroup(from, &flags);
  3366. ret = 0;
  3367. unlock:
  3368. unlock_page_cgroup(pc);
  3369. /*
  3370. * check events
  3371. */
  3372. memcg_check_events(to, page);
  3373. memcg_check_events(from, page);
  3374. out:
  3375. return ret;
  3376. }
  3377. /**
  3378. * mem_cgroup_move_parent - moves page to the parent group
  3379. * @page: the page to move
  3380. * @pc: page_cgroup of the page
  3381. * @child: page's cgroup
  3382. *
  3383. * move charges to its parent or the root cgroup if the group has no
  3384. * parent (aka use_hierarchy==0).
  3385. * Although this might fail (get_page_unless_zero, isolate_lru_page or
  3386. * mem_cgroup_move_account fails) the failure is always temporary and
  3387. * it signals a race with a page removal/uncharge or migration. In the
  3388. * first case the page is on the way out and it will vanish from the LRU
  3389. * on the next attempt and the call should be retried later.
  3390. * Isolation from the LRU fails only if page has been isolated from
  3391. * the LRU since we looked at it and that usually means either global
  3392. * reclaim or migration going on. The page will either get back to the
  3393. * LRU or vanish.
  3394. * Finaly mem_cgroup_move_account fails only if the page got uncharged
  3395. * (!PageCgroupUsed) or moved to a different group. The page will
  3396. * disappear in the next attempt.
  3397. */
  3398. static int mem_cgroup_move_parent(struct page *page,
  3399. struct page_cgroup *pc,
  3400. struct mem_cgroup *child)
  3401. {
  3402. struct mem_cgroup *parent;
  3403. unsigned int nr_pages;
  3404. unsigned long uninitialized_var(flags);
  3405. int ret;
  3406. VM_BUG_ON(mem_cgroup_is_root(child));
  3407. ret = -EBUSY;
  3408. if (!get_page_unless_zero(page))
  3409. goto out;
  3410. if (isolate_lru_page(page))
  3411. goto put;
  3412. nr_pages = hpage_nr_pages(page);
  3413. parent = parent_mem_cgroup(child);
  3414. /*
  3415. * If no parent, move charges to root cgroup.
  3416. */
  3417. if (!parent)
  3418. parent = root_mem_cgroup;
  3419. if (nr_pages > 1) {
  3420. VM_BUG_ON(!PageTransHuge(page));
  3421. flags = compound_lock_irqsave(page);
  3422. }
  3423. ret = mem_cgroup_move_account(page, nr_pages,
  3424. pc, child, parent);
  3425. if (!ret)
  3426. __mem_cgroup_cancel_local_charge(child, nr_pages);
  3427. if (nr_pages > 1)
  3428. compound_unlock_irqrestore(page, flags);
  3429. putback_lru_page(page);
  3430. put:
  3431. put_page(page);
  3432. out:
  3433. return ret;
  3434. }
  3435. /*
  3436. * Charge the memory controller for page usage.
  3437. * Return
  3438. * 0 if the charge was successful
  3439. * < 0 if the cgroup is over its limit
  3440. */
  3441. static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
  3442. gfp_t gfp_mask, enum charge_type ctype)
  3443. {
  3444. struct mem_cgroup *memcg = NULL;
  3445. unsigned int nr_pages = 1;
  3446. bool oom = true;
  3447. int ret;
  3448. if (PageTransHuge(page)) {
  3449. nr_pages <<= compound_order(page);
  3450. VM_BUG_ON(!PageTransHuge(page));
  3451. /*
  3452. * Never OOM-kill a process for a huge page. The
  3453. * fault handler will fall back to regular pages.
  3454. */
  3455. oom = false;
  3456. }
  3457. ret = __mem_cgroup_try_charge(mm, gfp_mask, nr_pages, &memcg, oom);
  3458. if (ret == -ENOMEM)
  3459. return ret;
  3460. __mem_cgroup_commit_charge(memcg, page, nr_pages, ctype, false);
  3461. return 0;
  3462. }
  3463. int mem_cgroup_newpage_charge(struct page *page,
  3464. struct mm_struct *mm, gfp_t gfp_mask)
  3465. {
  3466. if (mem_cgroup_disabled())
  3467. return 0;
  3468. VM_BUG_ON(page_mapped(page));
  3469. VM_BUG_ON(page->mapping && !PageAnon(page));
  3470. VM_BUG_ON(!mm);
  3471. return mem_cgroup_charge_common(page, mm, gfp_mask,
  3472. MEM_CGROUP_CHARGE_TYPE_ANON);
  3473. }
  3474. /*
  3475. * While swap-in, try_charge -> commit or cancel, the page is locked.
  3476. * And when try_charge() successfully returns, one refcnt to memcg without
  3477. * struct page_cgroup is acquired. This refcnt will be consumed by
  3478. * "commit()" or removed by "cancel()"
  3479. */
  3480. static int __mem_cgroup_try_charge_swapin(struct mm_struct *mm,
  3481. struct page *page,
  3482. gfp_t mask,
  3483. struct mem_cgroup **memcgp)
  3484. {
  3485. struct mem_cgroup *memcg;
  3486. struct page_cgroup *pc;
  3487. int ret;
  3488. pc = lookup_page_cgroup(page);
  3489. /*
  3490. * Every swap fault against a single page tries to charge the
  3491. * page, bail as early as possible. shmem_unuse() encounters
  3492. * already charged pages, too. The USED bit is protected by
  3493. * the page lock, which serializes swap cache removal, which
  3494. * in turn serializes uncharging.
  3495. */
  3496. if (PageCgroupUsed(pc))
  3497. return 0;
  3498. if (!do_swap_account)
  3499. goto charge_cur_mm;
  3500. memcg = try_get_mem_cgroup_from_page(page);
  3501. if (!memcg)
  3502. goto charge_cur_mm;
  3503. *memcgp = memcg;
  3504. ret = __mem_cgroup_try_charge(NULL, mask, 1, memcgp, true);
  3505. css_put(&memcg->css);
  3506. if (ret == -EINTR)
  3507. ret = 0;
  3508. return ret;
  3509. charge_cur_mm:
  3510. ret = __mem_cgroup_try_charge(mm, mask, 1, memcgp, true);
  3511. if (ret == -EINTR)
  3512. ret = 0;
  3513. return ret;
  3514. }
  3515. int mem_cgroup_try_charge_swapin(struct mm_struct *mm, struct page *page,
  3516. gfp_t gfp_mask, struct mem_cgroup **memcgp)
  3517. {
  3518. *memcgp = NULL;
  3519. if (mem_cgroup_disabled())
  3520. return 0;
  3521. /*
  3522. * A racing thread's fault, or swapoff, may have already
  3523. * updated the pte, and even removed page from swap cache: in
  3524. * those cases unuse_pte()'s pte_same() test will fail; but
  3525. * there's also a KSM case which does need to charge the page.
  3526. */
  3527. if (!PageSwapCache(page)) {
  3528. int ret;
  3529. ret = __mem_cgroup_try_charge(mm, gfp_mask, 1, memcgp, true);
  3530. if (ret == -EINTR)
  3531. ret = 0;
  3532. return ret;
  3533. }
  3534. return __mem_cgroup_try_charge_swapin(mm, page, gfp_mask, memcgp);
  3535. }
  3536. void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *memcg)
  3537. {
  3538. if (mem_cgroup_disabled())
  3539. return;
  3540. if (!memcg)
  3541. return;
  3542. __mem_cgroup_cancel_charge(memcg, 1);
  3543. }
  3544. static void
  3545. __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *memcg,
  3546. enum charge_type ctype)
  3547. {
  3548. if (mem_cgroup_disabled())
  3549. return;
  3550. if (!memcg)
  3551. return;
  3552. __mem_cgroup_commit_charge(memcg, page, 1, ctype, true);
  3553. /*
  3554. * Now swap is on-memory. This means this page may be
  3555. * counted both as mem and swap....double count.
  3556. * Fix it by uncharging from memsw. Basically, this SwapCache is stable
  3557. * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
  3558. * may call delete_from_swap_cache() before reach here.
  3559. */
  3560. if (do_swap_account && PageSwapCache(page)) {
  3561. swp_entry_t ent = {.val = page_private(page)};
  3562. mem_cgroup_uncharge_swap(ent);
  3563. }
  3564. }
  3565. void mem_cgroup_commit_charge_swapin(struct page *page,
  3566. struct mem_cgroup *memcg)
  3567. {
  3568. __mem_cgroup_commit_charge_swapin(page, memcg,
  3569. MEM_CGROUP_CHARGE_TYPE_ANON);
  3570. }
  3571. int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
  3572. gfp_t gfp_mask)
  3573. {
  3574. struct mem_cgroup *memcg = NULL;
  3575. enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
  3576. int ret;
  3577. if (mem_cgroup_disabled())
  3578. return 0;
  3579. if (PageCompound(page))
  3580. return 0;
  3581. if (!PageSwapCache(page))
  3582. ret = mem_cgroup_charge_common(page, mm, gfp_mask, type);
  3583. else { /* page is swapcache/shmem */
  3584. ret = __mem_cgroup_try_charge_swapin(mm, page,
  3585. gfp_mask, &memcg);
  3586. if (!ret)
  3587. __mem_cgroup_commit_charge_swapin(page, memcg, type);
  3588. }
  3589. return ret;
  3590. }
  3591. static void mem_cgroup_do_uncharge(struct mem_cgroup *memcg,
  3592. unsigned int nr_pages,
  3593. const enum charge_type ctype)
  3594. {
  3595. struct memcg_batch_info *batch = NULL;
  3596. bool uncharge_memsw = true;
  3597. /* If swapout, usage of swap doesn't decrease */
  3598. if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
  3599. uncharge_memsw = false;
  3600. batch = &current->memcg_batch;
  3601. /*
  3602. * In usual, we do css_get() when we remember memcg pointer.
  3603. * But in this case, we keep res->usage until end of a series of
  3604. * uncharges. Then, it's ok to ignore memcg's refcnt.
  3605. */
  3606. if (!batch->memcg)
  3607. batch->memcg = memcg;
  3608. /*
  3609. * do_batch > 0 when unmapping pages or inode invalidate/truncate.
  3610. * In those cases, all pages freed continuously can be expected to be in
  3611. * the same cgroup and we have chance to coalesce uncharges.
  3612. * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE)
  3613. * because we want to do uncharge as soon as possible.
  3614. */
  3615. if (!batch->do_batch || test_thread_flag(TIF_MEMDIE))
  3616. goto direct_uncharge;
  3617. if (nr_pages > 1)
  3618. goto direct_uncharge;
  3619. /*
  3620. * In typical case, batch->memcg == mem. This means we can
  3621. * merge a series of uncharges to an uncharge of res_counter.
  3622. * If not, we uncharge res_counter ony by one.
  3623. */
  3624. if (batch->memcg != memcg)
  3625. goto direct_uncharge;
  3626. /* remember freed charge and uncharge it later */
  3627. batch->nr_pages++;
  3628. if (uncharge_memsw)
  3629. batch->memsw_nr_pages++;
  3630. return;
  3631. direct_uncharge:
  3632. res_counter_uncharge(&memcg->res, nr_pages * PAGE_SIZE);
  3633. if (uncharge_memsw)
  3634. res_counter_uncharge(&memcg->memsw, nr_pages * PAGE_SIZE);
  3635. if (unlikely(batch->memcg != memcg))
  3636. memcg_oom_recover(memcg);
  3637. }
  3638. /*
  3639. * uncharge if !page_mapped(page)
  3640. */
  3641. static struct mem_cgroup *
  3642. __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype,
  3643. bool end_migration)
  3644. {
  3645. struct mem_cgroup *memcg = NULL;
  3646. unsigned int nr_pages = 1;
  3647. struct page_cgroup *pc;
  3648. bool anon;
  3649. if (mem_cgroup_disabled())
  3650. return NULL;
  3651. if (PageTransHuge(page)) {
  3652. nr_pages <<= compound_order(page);
  3653. VM_BUG_ON(!PageTransHuge(page));
  3654. }
  3655. /*
  3656. * Check if our page_cgroup is valid
  3657. */
  3658. pc = lookup_page_cgroup(page);
  3659. if (unlikely(!PageCgroupUsed(pc)))
  3660. return NULL;
  3661. lock_page_cgroup(pc);
  3662. memcg = pc->mem_cgroup;
  3663. if (!PageCgroupUsed(pc))
  3664. goto unlock_out;
  3665. anon = PageAnon(page);
  3666. switch (ctype) {
  3667. case MEM_CGROUP_CHARGE_TYPE_ANON:
  3668. /*
  3669. * Generally PageAnon tells if it's the anon statistics to be
  3670. * updated; but sometimes e.g. mem_cgroup_uncharge_page() is
  3671. * used before page reached the stage of being marked PageAnon.
  3672. */
  3673. anon = true;
  3674. /* fallthrough */
  3675. case MEM_CGROUP_CHARGE_TYPE_DROP:
  3676. /* See mem_cgroup_prepare_migration() */
  3677. if (page_mapped(page))
  3678. goto unlock_out;
  3679. /*
  3680. * Pages under migration may not be uncharged. But
  3681. * end_migration() /must/ be the one uncharging the
  3682. * unused post-migration page and so it has to call
  3683. * here with the migration bit still set. See the
  3684. * res_counter handling below.
  3685. */
  3686. if (!end_migration && PageCgroupMigration(pc))
  3687. goto unlock_out;
  3688. break;
  3689. case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
  3690. if (!PageAnon(page)) { /* Shared memory */
  3691. if (page->mapping && !page_is_file_cache(page))
  3692. goto unlock_out;
  3693. } else if (page_mapped(page)) /* Anon */
  3694. goto unlock_out;
  3695. break;
  3696. default:
  3697. break;
  3698. }
  3699. mem_cgroup_charge_statistics(memcg, page, anon, -nr_pages);
  3700. ClearPageCgroupUsed(pc);
  3701. /*
  3702. * pc->mem_cgroup is not cleared here. It will be accessed when it's
  3703. * freed from LRU. This is safe because uncharged page is expected not
  3704. * to be reused (freed soon). Exception is SwapCache, it's handled by
  3705. * special functions.
  3706. */
  3707. unlock_page_cgroup(pc);
  3708. /*
  3709. * even after unlock, we have memcg->res.usage here and this memcg
  3710. * will never be freed, so it's safe to call css_get().
  3711. */
  3712. memcg_check_events(memcg, page);
  3713. if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) {
  3714. mem_cgroup_swap_statistics(memcg, true);
  3715. css_get(&memcg->css);
  3716. }
  3717. /*
  3718. * Migration does not charge the res_counter for the
  3719. * replacement page, so leave it alone when phasing out the
  3720. * page that is unused after the migration.
  3721. */
  3722. if (!end_migration && !mem_cgroup_is_root(memcg))
  3723. mem_cgroup_do_uncharge(memcg, nr_pages, ctype);
  3724. return memcg;
  3725. unlock_out:
  3726. unlock_page_cgroup(pc);
  3727. return NULL;
  3728. }
  3729. void mem_cgroup_uncharge_page(struct page *page)
  3730. {
  3731. /* early check. */
  3732. if (page_mapped(page))
  3733. return;
  3734. VM_BUG_ON(page->mapping && !PageAnon(page));
  3735. /*
  3736. * If the page is in swap cache, uncharge should be deferred
  3737. * to the swap path, which also properly accounts swap usage
  3738. * and handles memcg lifetime.
  3739. *
  3740. * Note that this check is not stable and reclaim may add the
  3741. * page to swap cache at any time after this. However, if the
  3742. * page is not in swap cache by the time page->mapcount hits
  3743. * 0, there won't be any page table references to the swap
  3744. * slot, and reclaim will free it and not actually write the
  3745. * page to disk.
  3746. */
  3747. if (PageSwapCache(page))
  3748. return;
  3749. __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_ANON, false);
  3750. }
  3751. void mem_cgroup_uncharge_cache_page(struct page *page)
  3752. {
  3753. VM_BUG_ON(page_mapped(page));
  3754. VM_BUG_ON(page->mapping);
  3755. __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE, false);
  3756. }
  3757. /*
  3758. * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate.
  3759. * In that cases, pages are freed continuously and we can expect pages
  3760. * are in the same memcg. All these calls itself limits the number of
  3761. * pages freed at once, then uncharge_start/end() is called properly.
  3762. * This may be called prural(2) times in a context,
  3763. */
  3764. void mem_cgroup_uncharge_start(void)
  3765. {
  3766. current->memcg_batch.do_batch++;
  3767. /* We can do nest. */
  3768. if (current->memcg_batch.do_batch == 1) {
  3769. current->memcg_batch.memcg = NULL;
  3770. current->memcg_batch.nr_pages = 0;
  3771. current->memcg_batch.memsw_nr_pages = 0;
  3772. }
  3773. }
  3774. void mem_cgroup_uncharge_end(void)
  3775. {
  3776. struct memcg_batch_info *batch = &current->memcg_batch;
  3777. if (!batch->do_batch)
  3778. return;
  3779. batch->do_batch--;
  3780. if (batch->do_batch) /* If stacked, do nothing. */
  3781. return;
  3782. if (!batch->memcg)
  3783. return;
  3784. /*
  3785. * This "batch->memcg" is valid without any css_get/put etc...
  3786. * bacause we hide charges behind us.
  3787. */
  3788. if (batch->nr_pages)
  3789. res_counter_uncharge(&batch->memcg->res,
  3790. batch->nr_pages * PAGE_SIZE);
  3791. if (batch->memsw_nr_pages)
  3792. res_counter_uncharge(&batch->memcg->memsw,
  3793. batch->memsw_nr_pages * PAGE_SIZE);
  3794. memcg_oom_recover(batch->memcg);
  3795. /* forget this pointer (for sanity check) */
  3796. batch->memcg = NULL;
  3797. }
  3798. #ifdef CONFIG_SWAP
  3799. /*
  3800. * called after __delete_from_swap_cache() and drop "page" account.
  3801. * memcg information is recorded to swap_cgroup of "ent"
  3802. */
  3803. void
  3804. mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
  3805. {
  3806. struct mem_cgroup *memcg;
  3807. int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT;
  3808. if (!swapout) /* this was a swap cache but the swap is unused ! */
  3809. ctype = MEM_CGROUP_CHARGE_TYPE_DROP;
  3810. memcg = __mem_cgroup_uncharge_common(page, ctype, false);
  3811. /*
  3812. * record memcg information, if swapout && memcg != NULL,
  3813. * css_get() was called in uncharge().
  3814. */
  3815. if (do_swap_account && swapout && memcg)
  3816. swap_cgroup_record(ent, mem_cgroup_id(memcg));
  3817. }
  3818. #endif
  3819. #ifdef CONFIG_MEMCG_SWAP
  3820. /*
  3821. * called from swap_entry_free(). remove record in swap_cgroup and
  3822. * uncharge "memsw" account.
  3823. */
  3824. void mem_cgroup_uncharge_swap(swp_entry_t ent)
  3825. {
  3826. struct mem_cgroup *memcg;
  3827. unsigned short id;
  3828. if (!do_swap_account)
  3829. return;
  3830. id = swap_cgroup_record(ent, 0);
  3831. rcu_read_lock();
  3832. memcg = mem_cgroup_lookup(id);
  3833. if (memcg) {
  3834. /*
  3835. * We uncharge this because swap is freed.
  3836. * This memcg can be obsolete one. We avoid calling css_tryget
  3837. */
  3838. if (!mem_cgroup_is_root(memcg))
  3839. res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
  3840. mem_cgroup_swap_statistics(memcg, false);
  3841. css_put(&memcg->css);
  3842. }
  3843. rcu_read_unlock();
  3844. }
  3845. /**
  3846. * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
  3847. * @entry: swap entry to be moved
  3848. * @from: mem_cgroup which the entry is moved from
  3849. * @to: mem_cgroup which the entry is moved to
  3850. *
  3851. * It succeeds only when the swap_cgroup's record for this entry is the same
  3852. * as the mem_cgroup's id of @from.
  3853. *
  3854. * Returns 0 on success, -EINVAL on failure.
  3855. *
  3856. * The caller must have charged to @to, IOW, called res_counter_charge() about
  3857. * both res and memsw, and called css_get().
  3858. */
  3859. static int mem_cgroup_move_swap_account(swp_entry_t entry,
  3860. struct mem_cgroup *from, struct mem_cgroup *to)
  3861. {
  3862. unsigned short old_id, new_id;
  3863. old_id = mem_cgroup_id(from);
  3864. new_id = mem_cgroup_id(to);
  3865. if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
  3866. mem_cgroup_swap_statistics(from, false);
  3867. mem_cgroup_swap_statistics(to, true);
  3868. /*
  3869. * This function is only called from task migration context now.
  3870. * It postpones res_counter and refcount handling till the end
  3871. * of task migration(mem_cgroup_clear_mc()) for performance
  3872. * improvement. But we cannot postpone css_get(to) because if
  3873. * the process that has been moved to @to does swap-in, the
  3874. * refcount of @to might be decreased to 0.
  3875. *
  3876. * We are in attach() phase, so the cgroup is guaranteed to be
  3877. * alive, so we can just call css_get().
  3878. */
  3879. css_get(&to->css);
  3880. return 0;
  3881. }
  3882. return -EINVAL;
  3883. }
  3884. #else
  3885. static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
  3886. struct mem_cgroup *from, struct mem_cgroup *to)
  3887. {
  3888. return -EINVAL;
  3889. }
  3890. #endif
  3891. /*
  3892. * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
  3893. * page belongs to.
  3894. */
  3895. void mem_cgroup_prepare_migration(struct page *page, struct page *newpage,
  3896. struct mem_cgroup **memcgp)
  3897. {
  3898. struct mem_cgroup *memcg = NULL;
  3899. unsigned int nr_pages = 1;
  3900. struct page_cgroup *pc;
  3901. enum charge_type ctype;
  3902. *memcgp = NULL;
  3903. if (mem_cgroup_disabled())
  3904. return;
  3905. if (PageTransHuge(page))
  3906. nr_pages <<= compound_order(page);
  3907. pc = lookup_page_cgroup(page);
  3908. lock_page_cgroup(pc);
  3909. if (PageCgroupUsed(pc)) {
  3910. memcg = pc->mem_cgroup;
  3911. css_get(&memcg->css);
  3912. /*
  3913. * At migrating an anonymous page, its mapcount goes down
  3914. * to 0 and uncharge() will be called. But, even if it's fully
  3915. * unmapped, migration may fail and this page has to be
  3916. * charged again. We set MIGRATION flag here and delay uncharge
  3917. * until end_migration() is called
  3918. *
  3919. * Corner Case Thinking
  3920. * A)
  3921. * When the old page was mapped as Anon and it's unmap-and-freed
  3922. * while migration was ongoing.
  3923. * If unmap finds the old page, uncharge() of it will be delayed
  3924. * until end_migration(). If unmap finds a new page, it's
  3925. * uncharged when it make mapcount to be 1->0. If unmap code
  3926. * finds swap_migration_entry, the new page will not be mapped
  3927. * and end_migration() will find it(mapcount==0).
  3928. *
  3929. * B)
  3930. * When the old page was mapped but migraion fails, the kernel
  3931. * remaps it. A charge for it is kept by MIGRATION flag even
  3932. * if mapcount goes down to 0. We can do remap successfully
  3933. * without charging it again.
  3934. *
  3935. * C)
  3936. * The "old" page is under lock_page() until the end of
  3937. * migration, so, the old page itself will not be swapped-out.
  3938. * If the new page is swapped out before end_migraton, our
  3939. * hook to usual swap-out path will catch the event.
  3940. */
  3941. if (PageAnon(page))
  3942. SetPageCgroupMigration(pc);
  3943. }
  3944. unlock_page_cgroup(pc);
  3945. /*
  3946. * If the page is not charged at this point,
  3947. * we return here.
  3948. */
  3949. if (!memcg)
  3950. return;
  3951. *memcgp = memcg;
  3952. /*
  3953. * We charge new page before it's used/mapped. So, even if unlock_page()
  3954. * is called before end_migration, we can catch all events on this new
  3955. * page. In the case new page is migrated but not remapped, new page's
  3956. * mapcount will be finally 0 and we call uncharge in end_migration().
  3957. */
  3958. if (PageAnon(page))
  3959. ctype = MEM_CGROUP_CHARGE_TYPE_ANON;
  3960. else
  3961. ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
  3962. /*
  3963. * The page is committed to the memcg, but it's not actually
  3964. * charged to the res_counter since we plan on replacing the
  3965. * old one and only one page is going to be left afterwards.
  3966. */
  3967. __mem_cgroup_commit_charge(memcg, newpage, nr_pages, ctype, false);
  3968. }
  3969. /* remove redundant charge if migration failed*/
  3970. void mem_cgroup_end_migration(struct mem_cgroup *memcg,
  3971. struct page *oldpage, struct page *newpage, bool migration_ok)
  3972. {
  3973. struct page *used, *unused;
  3974. struct page_cgroup *pc;
  3975. bool anon;
  3976. if (!memcg)
  3977. return;
  3978. if (!migration_ok) {
  3979. used = oldpage;
  3980. unused = newpage;
  3981. } else {
  3982. used = newpage;
  3983. unused = oldpage;
  3984. }
  3985. anon = PageAnon(used);
  3986. __mem_cgroup_uncharge_common(unused,
  3987. anon ? MEM_CGROUP_CHARGE_TYPE_ANON
  3988. : MEM_CGROUP_CHARGE_TYPE_CACHE,
  3989. true);
  3990. css_put(&memcg->css);
  3991. /*
  3992. * We disallowed uncharge of pages under migration because mapcount
  3993. * of the page goes down to zero, temporarly.
  3994. * Clear the flag and check the page should be charged.
  3995. */
  3996. pc = lookup_page_cgroup(oldpage);
  3997. lock_page_cgroup(pc);
  3998. ClearPageCgroupMigration(pc);
  3999. unlock_page_cgroup(pc);
  4000. /*
  4001. * If a page is a file cache, radix-tree replacement is very atomic
  4002. * and we can skip this check. When it was an Anon page, its mapcount
  4003. * goes down to 0. But because we added MIGRATION flage, it's not
  4004. * uncharged yet. There are several case but page->mapcount check
  4005. * and USED bit check in mem_cgroup_uncharge_page() will do enough
  4006. * check. (see prepare_charge() also)
  4007. */
  4008. if (anon)
  4009. mem_cgroup_uncharge_page(used);
  4010. }
  4011. /*
  4012. * At replace page cache, newpage is not under any memcg but it's on
  4013. * LRU. So, this function doesn't touch res_counter but handles LRU
  4014. * in correct way. Both pages are locked so we cannot race with uncharge.
  4015. */
  4016. void mem_cgroup_replace_page_cache(struct page *oldpage,
  4017. struct page *newpage)
  4018. {
  4019. struct mem_cgroup *memcg = NULL;
  4020. struct page_cgroup *pc;
  4021. enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
  4022. if (mem_cgroup_disabled())
  4023. return;
  4024. pc = lookup_page_cgroup(oldpage);
  4025. /* fix accounting on old pages */
  4026. lock_page_cgroup(pc);
  4027. if (PageCgroupUsed(pc)) {
  4028. memcg = pc->mem_cgroup;
  4029. mem_cgroup_charge_statistics(memcg, oldpage, false, -1);
  4030. ClearPageCgroupUsed(pc);
  4031. }
  4032. unlock_page_cgroup(pc);
  4033. /*
  4034. * When called from shmem_replace_page(), in some cases the
  4035. * oldpage has already been charged, and in some cases not.
  4036. */
  4037. if (!memcg)
  4038. return;
  4039. /*
  4040. * Even if newpage->mapping was NULL before starting replacement,
  4041. * the newpage may be on LRU(or pagevec for LRU) already. We lock
  4042. * LRU while we overwrite pc->mem_cgroup.
  4043. */
  4044. __mem_cgroup_commit_charge(memcg, newpage, 1, type, true);
  4045. }
  4046. #ifdef CONFIG_DEBUG_VM
  4047. static struct page_cgroup *lookup_page_cgroup_used(struct page *page)
  4048. {
  4049. struct page_cgroup *pc;
  4050. pc = lookup_page_cgroup(page);
  4051. /*
  4052. * Can be NULL while feeding pages into the page allocator for
  4053. * the first time, i.e. during boot or memory hotplug;
  4054. * or when mem_cgroup_disabled().
  4055. */
  4056. if (likely(pc) && PageCgroupUsed(pc))
  4057. return pc;
  4058. return NULL;
  4059. }
  4060. bool mem_cgroup_bad_page_check(struct page *page)
  4061. {
  4062. if (mem_cgroup_disabled())
  4063. return false;
  4064. return lookup_page_cgroup_used(page) != NULL;
  4065. }
  4066. void mem_cgroup_print_bad_page(struct page *page)
  4067. {
  4068. struct page_cgroup *pc;
  4069. pc = lookup_page_cgroup_used(page);
  4070. if (pc) {
  4071. pr_alert("pc:%p pc->flags:%lx pc->mem_cgroup:%p\n",
  4072. pc, pc->flags, pc->mem_cgroup);
  4073. }
  4074. }
  4075. #endif
  4076. static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
  4077. unsigned long long val)
  4078. {
  4079. int retry_count;
  4080. u64 memswlimit, memlimit;
  4081. int ret = 0;
  4082. int children = mem_cgroup_count_children(memcg);
  4083. u64 curusage, oldusage;
  4084. int enlarge;
  4085. /*
  4086. * For keeping hierarchical_reclaim simple, how long we should retry
  4087. * is depends on callers. We set our retry-count to be function
  4088. * of # of children which we should visit in this loop.
  4089. */
  4090. retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;
  4091. oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
  4092. enlarge = 0;
  4093. while (retry_count) {
  4094. if (signal_pending(current)) {
  4095. ret = -EINTR;
  4096. break;
  4097. }
  4098. /*
  4099. * Rather than hide all in some function, I do this in
  4100. * open coded manner. You see what this really does.
  4101. * We have to guarantee memcg->res.limit <= memcg->memsw.limit.
  4102. */
  4103. mutex_lock(&set_limit_mutex);
  4104. memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
  4105. if (memswlimit < val) {
  4106. ret = -EINVAL;
  4107. mutex_unlock(&set_limit_mutex);
  4108. break;
  4109. }
  4110. memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
  4111. if (memlimit < val)
  4112. enlarge = 1;
  4113. ret = res_counter_set_limit(&memcg->res, val);
  4114. if (!ret) {
  4115. if (memswlimit == val)
  4116. memcg->memsw_is_minimum = true;
  4117. else
  4118. memcg->memsw_is_minimum = false;
  4119. }
  4120. mutex_unlock(&set_limit_mutex);
  4121. if (!ret)
  4122. break;
  4123. mem_cgroup_reclaim(memcg, GFP_KERNEL,
  4124. MEM_CGROUP_RECLAIM_SHRINK);
  4125. curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
  4126. /* Usage is reduced ? */
  4127. if (curusage >= oldusage)
  4128. retry_count--;
  4129. else
  4130. oldusage = curusage;
  4131. }
  4132. if (!ret && enlarge)
  4133. memcg_oom_recover(memcg);
  4134. return ret;
  4135. }
  4136. static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
  4137. unsigned long long val)
  4138. {
  4139. int retry_count;
  4140. u64 memlimit, memswlimit, oldusage, curusage;
  4141. int children = mem_cgroup_count_children(memcg);
  4142. int ret = -EBUSY;
  4143. int enlarge = 0;
  4144. /* see mem_cgroup_resize_res_limit */
  4145. retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
  4146. oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
  4147. while (retry_count) {
  4148. if (signal_pending(current)) {
  4149. ret = -EINTR;
  4150. break;
  4151. }
  4152. /*
  4153. * Rather than hide all in some function, I do this in
  4154. * open coded manner. You see what this really does.
  4155. * We have to guarantee memcg->res.limit <= memcg->memsw.limit.
  4156. */
  4157. mutex_lock(&set_limit_mutex);
  4158. memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
  4159. if (memlimit > val) {
  4160. ret = -EINVAL;
  4161. mutex_unlock(&set_limit_mutex);
  4162. break;
  4163. }
  4164. memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
  4165. if (memswlimit < val)
  4166. enlarge = 1;
  4167. ret = res_counter_set_limit(&memcg->memsw, val);
  4168. if (!ret) {
  4169. if (memlimit == val)
  4170. memcg->memsw_is_minimum = true;
  4171. else
  4172. memcg->memsw_is_minimum = false;
  4173. }
  4174. mutex_unlock(&set_limit_mutex);
  4175. if (!ret)
  4176. break;
  4177. mem_cgroup_reclaim(memcg, GFP_KERNEL,
  4178. MEM_CGROUP_RECLAIM_NOSWAP |
  4179. MEM_CGROUP_RECLAIM_SHRINK);
  4180. curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
  4181. /* Usage is reduced ? */
  4182. if (curusage >= oldusage)
  4183. retry_count--;
  4184. else
  4185. oldusage = curusage;
  4186. }
  4187. if (!ret && enlarge)
  4188. memcg_oom_recover(memcg);
  4189. return ret;
  4190. }
  4191. unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
  4192. gfp_t gfp_mask,
  4193. unsigned long *total_scanned)
  4194. {
  4195. unsigned long nr_reclaimed = 0;
  4196. struct mem_cgroup_per_zone *mz, *next_mz = NULL;
  4197. unsigned long reclaimed;
  4198. int loop = 0;
  4199. struct mem_cgroup_tree_per_zone *mctz;
  4200. unsigned long long excess;
  4201. unsigned long nr_scanned;
  4202. if (order > 0)
  4203. return 0;
  4204. mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone));
  4205. /*
  4206. * This loop can run a while, specially if mem_cgroup's continuously
  4207. * keep exceeding their soft limit and putting the system under
  4208. * pressure
  4209. */
  4210. do {
  4211. if (next_mz)
  4212. mz = next_mz;
  4213. else
  4214. mz = mem_cgroup_largest_soft_limit_node(mctz);
  4215. if (!mz)
  4216. break;
  4217. nr_scanned = 0;
  4218. reclaimed = mem_cgroup_soft_reclaim(mz->memcg, zone,
  4219. gfp_mask, &nr_scanned);
  4220. nr_reclaimed += reclaimed;
  4221. *total_scanned += nr_scanned;
  4222. spin_lock(&mctz->lock);
  4223. /*
  4224. * If we failed to reclaim anything from this memory cgroup
  4225. * it is time to move on to the next cgroup
  4226. */
  4227. next_mz = NULL;
  4228. if (!reclaimed) {
  4229. do {
  4230. /*
  4231. * Loop until we find yet another one.
  4232. *
  4233. * By the time we get the soft_limit lock
  4234. * again, someone might have aded the
  4235. * group back on the RB tree. Iterate to
  4236. * make sure we get a different mem.
  4237. * mem_cgroup_largest_soft_limit_node returns
  4238. * NULL if no other cgroup is present on
  4239. * the tree
  4240. */
  4241. next_mz =
  4242. __mem_cgroup_largest_soft_limit_node(mctz);
  4243. if (next_mz == mz)
  4244. css_put(&next_mz->memcg->css);
  4245. else /* next_mz == NULL or other memcg */
  4246. break;
  4247. } while (1);
  4248. }
  4249. __mem_cgroup_remove_exceeded(mz->memcg, mz, mctz);
  4250. excess = res_counter_soft_limit_excess(&mz->memcg->res);
  4251. /*
  4252. * One school of thought says that we should not add
  4253. * back the node to the tree if reclaim returns 0.
  4254. * But our reclaim could return 0, simply because due
  4255. * to priority we are exposing a smaller subset of
  4256. * memory to reclaim from. Consider this as a longer
  4257. * term TODO.
  4258. */
  4259. /* If excess == 0, no tree ops */
  4260. __mem_cgroup_insert_exceeded(mz->memcg, mz, mctz, excess);
  4261. spin_unlock(&mctz->lock);
  4262. css_put(&mz->memcg->css);
  4263. loop++;
  4264. /*
  4265. * Could not reclaim anything and there are no more
  4266. * mem cgroups to try or we seem to be looping without
  4267. * reclaiming anything.
  4268. */
  4269. if (!nr_reclaimed &&
  4270. (next_mz == NULL ||
  4271. loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
  4272. break;
  4273. } while (!nr_reclaimed);
  4274. if (next_mz)
  4275. css_put(&next_mz->memcg->css);
  4276. return nr_reclaimed;
  4277. }
  4278. /**
  4279. * mem_cgroup_force_empty_list - clears LRU of a group
  4280. * @memcg: group to clear
  4281. * @node: NUMA node
  4282. * @zid: zone id
  4283. * @lru: lru to to clear
  4284. *
  4285. * Traverse a specified page_cgroup list and try to drop them all. This doesn't
  4286. * reclaim the pages page themselves - pages are moved to the parent (or root)
  4287. * group.
  4288. */
  4289. static void mem_cgroup_force_empty_list(struct mem_cgroup *memcg,
  4290. int node, int zid, enum lru_list lru)
  4291. {
  4292. struct lruvec *lruvec;
  4293. unsigned long flags;
  4294. struct list_head *list;
  4295. struct page *busy;
  4296. struct zone *zone;
  4297. zone = &NODE_DATA(node)->node_zones[zid];
  4298. lruvec = mem_cgroup_zone_lruvec(zone, memcg);
  4299. list = &lruvec->lists[lru];
  4300. busy = NULL;
  4301. do {
  4302. struct page_cgroup *pc;
  4303. struct page *page;
  4304. spin_lock_irqsave(&zone->lru_lock, flags);
  4305. if (list_empty(list)) {
  4306. spin_unlock_irqrestore(&zone->lru_lock, flags);
  4307. break;
  4308. }
  4309. page = list_entry(list->prev, struct page, lru);
  4310. if (busy == page) {
  4311. list_move(&page->lru, list);
  4312. busy = NULL;
  4313. spin_unlock_irqrestore(&zone->lru_lock, flags);
  4314. continue;
  4315. }
  4316. spin_unlock_irqrestore(&zone->lru_lock, flags);
  4317. pc = lookup_page_cgroup(page);
  4318. if (mem_cgroup_move_parent(page, pc, memcg)) {
  4319. /* found lock contention or "pc" is obsolete. */
  4320. busy = page;
  4321. cond_resched();
  4322. } else
  4323. busy = NULL;
  4324. } while (!list_empty(list));
  4325. }
  4326. /*
  4327. * make mem_cgroup's charge to be 0 if there is no task by moving
  4328. * all the charges and pages to the parent.
  4329. * This enables deleting this mem_cgroup.
  4330. *
  4331. * Caller is responsible for holding css reference on the memcg.
  4332. */
  4333. static void mem_cgroup_reparent_charges(struct mem_cgroup *memcg)
  4334. {
  4335. int node, zid;
  4336. u64 usage;
  4337. do {
  4338. /* This is for making all *used* pages to be on LRU. */
  4339. lru_add_drain_all();
  4340. drain_all_stock_sync(memcg);
  4341. mem_cgroup_start_move(memcg);
  4342. for_each_node_state(node, N_MEMORY) {
  4343. for (zid = 0; zid < MAX_NR_ZONES; zid++) {
  4344. enum lru_list lru;
  4345. for_each_lru(lru) {
  4346. mem_cgroup_force_empty_list(memcg,
  4347. node, zid, lru);
  4348. }
  4349. }
  4350. }
  4351. mem_cgroup_end_move(memcg);
  4352. memcg_oom_recover(memcg);
  4353. cond_resched();
  4354. /*
  4355. * Kernel memory may not necessarily be trackable to a specific
  4356. * process. So they are not migrated, and therefore we can't
  4357. * expect their value to drop to 0 here.
  4358. * Having res filled up with kmem only is enough.
  4359. *
  4360. * This is a safety check because mem_cgroup_force_empty_list
  4361. * could have raced with mem_cgroup_replace_page_cache callers
  4362. * so the lru seemed empty but the page could have been added
  4363. * right after the check. RES_USAGE should be safe as we always
  4364. * charge before adding to the LRU.
  4365. */
  4366. usage = res_counter_read_u64(&memcg->res, RES_USAGE) -
  4367. res_counter_read_u64(&memcg->kmem, RES_USAGE);
  4368. } while (usage > 0);
  4369. }
  4370. static inline bool memcg_has_children(struct mem_cgroup *memcg)
  4371. {
  4372. lockdep_assert_held(&memcg_create_mutex);
  4373. /*
  4374. * The lock does not prevent addition or deletion to the list
  4375. * of children, but it prevents a new child from being
  4376. * initialized based on this parent in css_online(), so it's
  4377. * enough to decide whether hierarchically inherited
  4378. * attributes can still be changed or not.
  4379. */
  4380. return memcg->use_hierarchy &&
  4381. !list_empty(&memcg->css.cgroup->children);
  4382. }
  4383. /*
  4384. * Reclaims as many pages from the given memcg as possible and moves
  4385. * the rest to the parent.
  4386. *
  4387. * Caller is responsible for holding css reference for memcg.
  4388. */
  4389. static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
  4390. {
  4391. int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
  4392. struct cgroup *cgrp = memcg->css.cgroup;
  4393. /* returns EBUSY if there is a task or if we come here twice. */
  4394. if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
  4395. return -EBUSY;
  4396. /* we call try-to-free pages for make this cgroup empty */
  4397. lru_add_drain_all();
  4398. /* try to free all pages in this cgroup */
  4399. while (nr_retries && res_counter_read_u64(&memcg->res, RES_USAGE) > 0) {
  4400. int progress;
  4401. if (signal_pending(current))
  4402. return -EINTR;
  4403. progress = try_to_free_mem_cgroup_pages(memcg, GFP_KERNEL,
  4404. false);
  4405. if (!progress) {
  4406. nr_retries--;
  4407. /* maybe some writeback is necessary */
  4408. congestion_wait(BLK_RW_ASYNC, HZ/10);
  4409. }
  4410. }
  4411. lru_add_drain();
  4412. mem_cgroup_reparent_charges(memcg);
  4413. return 0;
  4414. }
  4415. static int mem_cgroup_force_empty_write(struct cgroup_subsys_state *css,
  4416. unsigned int event)
  4417. {
  4418. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  4419. if (mem_cgroup_is_root(memcg))
  4420. return -EINVAL;
  4421. return mem_cgroup_force_empty(memcg);
  4422. }
  4423. static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
  4424. struct cftype *cft)
  4425. {
  4426. return mem_cgroup_from_css(css)->use_hierarchy;
  4427. }
  4428. static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
  4429. struct cftype *cft, u64 val)
  4430. {
  4431. int retval = 0;
  4432. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  4433. struct mem_cgroup *parent_memcg = mem_cgroup_from_css(css_parent(&memcg->css));
  4434. mutex_lock(&memcg_create_mutex);
  4435. if (memcg->use_hierarchy == val)
  4436. goto out;
  4437. /*
  4438. * If parent's use_hierarchy is set, we can't make any modifications
  4439. * in the child subtrees. If it is unset, then the change can
  4440. * occur, provided the current cgroup has no children.
  4441. *
  4442. * For the root cgroup, parent_mem is NULL, we allow value to be
  4443. * set if there are no children.
  4444. */
  4445. if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
  4446. (val == 1 || val == 0)) {
  4447. if (list_empty(&memcg->css.cgroup->children))
  4448. memcg->use_hierarchy = val;
  4449. else
  4450. retval = -EBUSY;
  4451. } else
  4452. retval = -EINVAL;
  4453. out:
  4454. mutex_unlock(&memcg_create_mutex);
  4455. return retval;
  4456. }
  4457. static unsigned long mem_cgroup_recursive_stat(struct mem_cgroup *memcg,
  4458. enum mem_cgroup_stat_index idx)
  4459. {
  4460. struct mem_cgroup *iter;
  4461. long val = 0;
  4462. /* Per-cpu values can be negative, use a signed accumulator */
  4463. for_each_mem_cgroup_tree(iter, memcg)
  4464. val += mem_cgroup_read_stat(iter, idx);
  4465. if (val < 0) /* race ? */
  4466. val = 0;
  4467. return val;
  4468. }
  4469. static inline u64 mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
  4470. {
  4471. u64 val;
  4472. if (!mem_cgroup_is_root(memcg)) {
  4473. if (!swap)
  4474. return res_counter_read_u64(&memcg->res, RES_USAGE);
  4475. else
  4476. return res_counter_read_u64(&memcg->memsw, RES_USAGE);
  4477. }
  4478. /*
  4479. * Transparent hugepages are still accounted for in MEM_CGROUP_STAT_RSS
  4480. * as well as in MEM_CGROUP_STAT_RSS_HUGE.
  4481. */
  4482. val = mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_CACHE);
  4483. val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_RSS);
  4484. if (swap)
  4485. val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_SWAP);
  4486. return val << PAGE_SHIFT;
  4487. }
  4488. static ssize_t mem_cgroup_read(struct cgroup_subsys_state *css,
  4489. struct cftype *cft, struct file *file,
  4490. char __user *buf, size_t nbytes, loff_t *ppos)
  4491. {
  4492. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  4493. char str[64];
  4494. u64 val;
  4495. int name, len;
  4496. enum res_type type;
  4497. type = MEMFILE_TYPE(cft->private);
  4498. name = MEMFILE_ATTR(cft->private);
  4499. switch (type) {
  4500. case _MEM:
  4501. if (name == RES_USAGE)
  4502. val = mem_cgroup_usage(memcg, false);
  4503. else
  4504. val = res_counter_read_u64(&memcg->res, name);
  4505. break;
  4506. case _MEMSWAP:
  4507. if (name == RES_USAGE)
  4508. val = mem_cgroup_usage(memcg, true);
  4509. else
  4510. val = res_counter_read_u64(&memcg->memsw, name);
  4511. break;
  4512. case _KMEM:
  4513. val = res_counter_read_u64(&memcg->kmem, name);
  4514. break;
  4515. default:
  4516. BUG();
  4517. }
  4518. len = scnprintf(str, sizeof(str), "%llu\n", (unsigned long long)val);
  4519. return simple_read_from_buffer(buf, nbytes, ppos, str, len);
  4520. }
  4521. static int memcg_update_kmem_limit(struct cgroup_subsys_state *css, u64 val)
  4522. {
  4523. int ret = -EINVAL;
  4524. #ifdef CONFIG_MEMCG_KMEM
  4525. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  4526. /*
  4527. * For simplicity, we won't allow this to be disabled. It also can't
  4528. * be changed if the cgroup has children already, or if tasks had
  4529. * already joined.
  4530. *
  4531. * If tasks join before we set the limit, a person looking at
  4532. * kmem.usage_in_bytes will have no way to determine when it took
  4533. * place, which makes the value quite meaningless.
  4534. *
  4535. * After it first became limited, changes in the value of the limit are
  4536. * of course permitted.
  4537. */
  4538. mutex_lock(&memcg_create_mutex);
  4539. mutex_lock(&set_limit_mutex);
  4540. if (!memcg->kmem_account_flags && val != RES_COUNTER_MAX) {
  4541. if (cgroup_task_count(css->cgroup) || memcg_has_children(memcg)) {
  4542. ret = -EBUSY;
  4543. goto out;
  4544. }
  4545. ret = res_counter_set_limit(&memcg->kmem, val);
  4546. VM_BUG_ON(ret);
  4547. ret = memcg_update_cache_sizes(memcg);
  4548. if (ret) {
  4549. res_counter_set_limit(&memcg->kmem, RES_COUNTER_MAX);
  4550. goto out;
  4551. }
  4552. static_key_slow_inc(&memcg_kmem_enabled_key);
  4553. /*
  4554. * setting the active bit after the inc will guarantee no one
  4555. * starts accounting before all call sites are patched
  4556. */
  4557. memcg_kmem_set_active(memcg);
  4558. } else
  4559. ret = res_counter_set_limit(&memcg->kmem, val);
  4560. out:
  4561. mutex_unlock(&set_limit_mutex);
  4562. mutex_unlock(&memcg_create_mutex);
  4563. #endif
  4564. return ret;
  4565. }
  4566. #ifdef CONFIG_MEMCG_KMEM
  4567. static int memcg_propagate_kmem(struct mem_cgroup *memcg)
  4568. {
  4569. int ret = 0;
  4570. struct mem_cgroup *parent = parent_mem_cgroup(memcg);
  4571. if (!parent)
  4572. goto out;
  4573. memcg->kmem_account_flags = parent->kmem_account_flags;
  4574. /*
  4575. * When that happen, we need to disable the static branch only on those
  4576. * memcgs that enabled it. To achieve this, we would be forced to
  4577. * complicate the code by keeping track of which memcgs were the ones
  4578. * that actually enabled limits, and which ones got it from its
  4579. * parents.
  4580. *
  4581. * It is a lot simpler just to do static_key_slow_inc() on every child
  4582. * that is accounted.
  4583. */
  4584. if (!memcg_kmem_is_active(memcg))
  4585. goto out;
  4586. /*
  4587. * __mem_cgroup_free() will issue static_key_slow_dec() because this
  4588. * memcg is active already. If the later initialization fails then the
  4589. * cgroup core triggers the cleanup so we do not have to do it here.
  4590. */
  4591. static_key_slow_inc(&memcg_kmem_enabled_key);
  4592. mutex_lock(&set_limit_mutex);
  4593. memcg_stop_kmem_account();
  4594. ret = memcg_update_cache_sizes(memcg);
  4595. memcg_resume_kmem_account();
  4596. mutex_unlock(&set_limit_mutex);
  4597. out:
  4598. return ret;
  4599. }
  4600. #endif /* CONFIG_MEMCG_KMEM */
  4601. /*
  4602. * The user of this function is...
  4603. * RES_LIMIT.
  4604. */
  4605. static int mem_cgroup_write(struct cgroup_subsys_state *css, struct cftype *cft,
  4606. const char *buffer)
  4607. {
  4608. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  4609. enum res_type type;
  4610. int name;
  4611. unsigned long long val;
  4612. int ret;
  4613. type = MEMFILE_TYPE(cft->private);
  4614. name = MEMFILE_ATTR(cft->private);
  4615. switch (name) {
  4616. case RES_LIMIT:
  4617. if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
  4618. ret = -EINVAL;
  4619. break;
  4620. }
  4621. /* This function does all necessary parse...reuse it */
  4622. ret = res_counter_memparse_write_strategy(buffer, &val);
  4623. if (ret)
  4624. break;
  4625. if (type == _MEM)
  4626. ret = mem_cgroup_resize_limit(memcg, val);
  4627. else if (type == _MEMSWAP)
  4628. ret = mem_cgroup_resize_memsw_limit(memcg, val);
  4629. else if (type == _KMEM)
  4630. ret = memcg_update_kmem_limit(css, val);
  4631. else
  4632. return -EINVAL;
  4633. break;
  4634. case RES_SOFT_LIMIT:
  4635. ret = res_counter_memparse_write_strategy(buffer, &val);
  4636. if (ret)
  4637. break;
  4638. /*
  4639. * For memsw, soft limits are hard to implement in terms
  4640. * of semantics, for now, we support soft limits for
  4641. * control without swap
  4642. */
  4643. if (type == _MEM)
  4644. ret = res_counter_set_soft_limit(&memcg->res, val);
  4645. else
  4646. ret = -EINVAL;
  4647. break;
  4648. default:
  4649. ret = -EINVAL; /* should be BUG() ? */
  4650. break;
  4651. }
  4652. return ret;
  4653. }
  4654. static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg,
  4655. unsigned long long *mem_limit, unsigned long long *memsw_limit)
  4656. {
  4657. unsigned long long min_limit, min_memsw_limit, tmp;
  4658. min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
  4659. min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
  4660. if (!memcg->use_hierarchy)
  4661. goto out;
  4662. while (css_parent(&memcg->css)) {
  4663. memcg = mem_cgroup_from_css(css_parent(&memcg->css));
  4664. if (!memcg->use_hierarchy)
  4665. break;
  4666. tmp = res_counter_read_u64(&memcg->res, RES_LIMIT);
  4667. min_limit = min(min_limit, tmp);
  4668. tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
  4669. min_memsw_limit = min(min_memsw_limit, tmp);
  4670. }
  4671. out:
  4672. *mem_limit = min_limit;
  4673. *memsw_limit = min_memsw_limit;
  4674. }
  4675. static int mem_cgroup_reset(struct cgroup_subsys_state *css, unsigned int event)
  4676. {
  4677. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  4678. int name;
  4679. enum res_type type;
  4680. type = MEMFILE_TYPE(event);
  4681. name = MEMFILE_ATTR(event);
  4682. switch (name) {
  4683. case RES_MAX_USAGE:
  4684. if (type == _MEM)
  4685. res_counter_reset_max(&memcg->res);
  4686. else if (type == _MEMSWAP)
  4687. res_counter_reset_max(&memcg->memsw);
  4688. else if (type == _KMEM)
  4689. res_counter_reset_max(&memcg->kmem);
  4690. else
  4691. return -EINVAL;
  4692. break;
  4693. case RES_FAILCNT:
  4694. if (type == _MEM)
  4695. res_counter_reset_failcnt(&memcg->res);
  4696. else if (type == _MEMSWAP)
  4697. res_counter_reset_failcnt(&memcg->memsw);
  4698. else if (type == _KMEM)
  4699. res_counter_reset_failcnt(&memcg->kmem);
  4700. else
  4701. return -EINVAL;
  4702. break;
  4703. }
  4704. return 0;
  4705. }
  4706. static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
  4707. struct cftype *cft)
  4708. {
  4709. return mem_cgroup_from_css(css)->move_charge_at_immigrate;
  4710. }
  4711. #ifdef CONFIG_MMU
  4712. static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
  4713. struct cftype *cft, u64 val)
  4714. {
  4715. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  4716. if (val >= (1 << NR_MOVE_TYPE))
  4717. return -EINVAL;
  4718. /*
  4719. * No kind of locking is needed in here, because ->can_attach() will
  4720. * check this value once in the beginning of the process, and then carry
  4721. * on with stale data. This means that changes to this value will only
  4722. * affect task migrations starting after the change.
  4723. */
  4724. memcg->move_charge_at_immigrate = val;
  4725. return 0;
  4726. }
  4727. #else
  4728. static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
  4729. struct cftype *cft, u64 val)
  4730. {
  4731. return -ENOSYS;
  4732. }
  4733. #endif
  4734. #ifdef CONFIG_NUMA
  4735. static int memcg_numa_stat_show(struct cgroup_subsys_state *css,
  4736. struct cftype *cft, struct seq_file *m)
  4737. {
  4738. struct numa_stat {
  4739. const char *name;
  4740. unsigned int lru_mask;
  4741. };
  4742. static const struct numa_stat stats[] = {
  4743. { "total", LRU_ALL },
  4744. { "file", LRU_ALL_FILE },
  4745. { "anon", LRU_ALL_ANON },
  4746. { "unevictable", BIT(LRU_UNEVICTABLE) },
  4747. };
  4748. const struct numa_stat *stat;
  4749. int nid;
  4750. unsigned long nr;
  4751. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  4752. for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
  4753. nr = mem_cgroup_nr_lru_pages(memcg, stat->lru_mask);
  4754. seq_printf(m, "%s=%lu", stat->name, nr);
  4755. for_each_node_state(nid, N_MEMORY) {
  4756. nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
  4757. stat->lru_mask);
  4758. seq_printf(m, " N%d=%lu", nid, nr);
  4759. }
  4760. seq_putc(m, '\n');
  4761. }
  4762. for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
  4763. struct mem_cgroup *iter;
  4764. nr = 0;
  4765. for_each_mem_cgroup_tree(iter, memcg)
  4766. nr += mem_cgroup_nr_lru_pages(iter, stat->lru_mask);
  4767. seq_printf(m, "hierarchical_%s=%lu", stat->name, nr);
  4768. for_each_node_state(nid, N_MEMORY) {
  4769. nr = 0;
  4770. for_each_mem_cgroup_tree(iter, memcg)
  4771. nr += mem_cgroup_node_nr_lru_pages(
  4772. iter, nid, stat->lru_mask);
  4773. seq_printf(m, " N%d=%lu", nid, nr);
  4774. }
  4775. seq_putc(m, '\n');
  4776. }
  4777. return 0;
  4778. }
  4779. #endif /* CONFIG_NUMA */
  4780. static inline void mem_cgroup_lru_names_not_uptodate(void)
  4781. {
  4782. BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);
  4783. }
  4784. static int memcg_stat_show(struct cgroup_subsys_state *css, struct cftype *cft,
  4785. struct seq_file *m)
  4786. {
  4787. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  4788. struct mem_cgroup *mi;
  4789. unsigned int i;
  4790. for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
  4791. if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
  4792. continue;
  4793. seq_printf(m, "%s %ld\n", mem_cgroup_stat_names[i],
  4794. mem_cgroup_read_stat(memcg, i) * PAGE_SIZE);
  4795. }
  4796. for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++)
  4797. seq_printf(m, "%s %lu\n", mem_cgroup_events_names[i],
  4798. mem_cgroup_read_events(memcg, i));
  4799. for (i = 0; i < NR_LRU_LISTS; i++)
  4800. seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
  4801. mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);
  4802. /* Hierarchical information */
  4803. {
  4804. unsigned long long limit, memsw_limit;
  4805. memcg_get_hierarchical_limit(memcg, &limit, &memsw_limit);
  4806. seq_printf(m, "hierarchical_memory_limit %llu\n", limit);
  4807. if (do_swap_account)
  4808. seq_printf(m, "hierarchical_memsw_limit %llu\n",
  4809. memsw_limit);
  4810. }
  4811. for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
  4812. long long val = 0;
  4813. if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
  4814. continue;
  4815. for_each_mem_cgroup_tree(mi, memcg)
  4816. val += mem_cgroup_read_stat(mi, i) * PAGE_SIZE;
  4817. seq_printf(m, "total_%s %lld\n", mem_cgroup_stat_names[i], val);
  4818. }
  4819. for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
  4820. unsigned long long val = 0;
  4821. for_each_mem_cgroup_tree(mi, memcg)
  4822. val += mem_cgroup_read_events(mi, i);
  4823. seq_printf(m, "total_%s %llu\n",
  4824. mem_cgroup_events_names[i], val);
  4825. }
  4826. for (i = 0; i < NR_LRU_LISTS; i++) {
  4827. unsigned long long val = 0;
  4828. for_each_mem_cgroup_tree(mi, memcg)
  4829. val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE;
  4830. seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val);
  4831. }
  4832. #ifdef CONFIG_DEBUG_VM
  4833. {
  4834. int nid, zid;
  4835. struct mem_cgroup_per_zone *mz;
  4836. struct zone_reclaim_stat *rstat;
  4837. unsigned long recent_rotated[2] = {0, 0};
  4838. unsigned long recent_scanned[2] = {0, 0};
  4839. for_each_online_node(nid)
  4840. for (zid = 0; zid < MAX_NR_ZONES; zid++) {
  4841. mz = mem_cgroup_zoneinfo(memcg, nid, zid);
  4842. rstat = &mz->lruvec.reclaim_stat;
  4843. recent_rotated[0] += rstat->recent_rotated[0];
  4844. recent_rotated[1] += rstat->recent_rotated[1];
  4845. recent_scanned[0] += rstat->recent_scanned[0];
  4846. recent_scanned[1] += rstat->recent_scanned[1];
  4847. }
  4848. seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
  4849. seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
  4850. seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
  4851. seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
  4852. }
  4853. #endif
  4854. return 0;
  4855. }
  4856. static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
  4857. struct cftype *cft)
  4858. {
  4859. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  4860. return mem_cgroup_swappiness(memcg);
  4861. }
  4862. static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
  4863. struct cftype *cft, u64 val)
  4864. {
  4865. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  4866. struct mem_cgroup *parent = mem_cgroup_from_css(css_parent(&memcg->css));
  4867. if (val > 100 || !parent)
  4868. return -EINVAL;
  4869. mutex_lock(&memcg_create_mutex);
  4870. /* If under hierarchy, only empty-root can set this value */
  4871. if ((parent->use_hierarchy) || memcg_has_children(memcg)) {
  4872. mutex_unlock(&memcg_create_mutex);
  4873. return -EINVAL;
  4874. }
  4875. memcg->swappiness = val;
  4876. mutex_unlock(&memcg_create_mutex);
  4877. return 0;
  4878. }
  4879. static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
  4880. {
  4881. struct mem_cgroup_threshold_ary *t;
  4882. u64 usage;
  4883. int i;
  4884. rcu_read_lock();
  4885. if (!swap)
  4886. t = rcu_dereference(memcg->thresholds.primary);
  4887. else
  4888. t = rcu_dereference(memcg->memsw_thresholds.primary);
  4889. if (!t)
  4890. goto unlock;
  4891. usage = mem_cgroup_usage(memcg, swap);
  4892. /*
  4893. * current_threshold points to threshold just below or equal to usage.
  4894. * If it's not true, a threshold was crossed after last
  4895. * call of __mem_cgroup_threshold().
  4896. */
  4897. i = t->current_threshold;
  4898. /*
  4899. * Iterate backward over array of thresholds starting from
  4900. * current_threshold and check if a threshold is crossed.
  4901. * If none of thresholds below usage is crossed, we read
  4902. * only one element of the array here.
  4903. */
  4904. for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
  4905. eventfd_signal(t->entries[i].eventfd, 1);
  4906. /* i = current_threshold + 1 */
  4907. i++;
  4908. /*
  4909. * Iterate forward over array of thresholds starting from
  4910. * current_threshold+1 and check if a threshold is crossed.
  4911. * If none of thresholds above usage is crossed, we read
  4912. * only one element of the array here.
  4913. */
  4914. for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
  4915. eventfd_signal(t->entries[i].eventfd, 1);
  4916. /* Update current_threshold */
  4917. t->current_threshold = i - 1;
  4918. unlock:
  4919. rcu_read_unlock();
  4920. }
  4921. static void mem_cgroup_threshold(struct mem_cgroup *memcg)
  4922. {
  4923. while (memcg) {
  4924. __mem_cgroup_threshold(memcg, false);
  4925. if (do_swap_account)
  4926. __mem_cgroup_threshold(memcg, true);
  4927. memcg = parent_mem_cgroup(memcg);
  4928. }
  4929. }
  4930. static int compare_thresholds(const void *a, const void *b)
  4931. {
  4932. const struct mem_cgroup_threshold *_a = a;
  4933. const struct mem_cgroup_threshold *_b = b;
  4934. if (_a->threshold > _b->threshold)
  4935. return 1;
  4936. if (_a->threshold < _b->threshold)
  4937. return -1;
  4938. return 0;
  4939. }
  4940. static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
  4941. {
  4942. struct mem_cgroup_eventfd_list *ev;
  4943. list_for_each_entry(ev, &memcg->oom_notify, list)
  4944. eventfd_signal(ev->eventfd, 1);
  4945. return 0;
  4946. }
  4947. static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
  4948. {
  4949. struct mem_cgroup *iter;
  4950. for_each_mem_cgroup_tree(iter, memcg)
  4951. mem_cgroup_oom_notify_cb(iter);
  4952. }
  4953. static int mem_cgroup_usage_register_event(struct cgroup_subsys_state *css,
  4954. struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
  4955. {
  4956. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  4957. struct mem_cgroup_thresholds *thresholds;
  4958. struct mem_cgroup_threshold_ary *new;
  4959. enum res_type type = MEMFILE_TYPE(cft->private);
  4960. u64 threshold, usage;
  4961. int i, size, ret;
  4962. ret = res_counter_memparse_write_strategy(args, &threshold);
  4963. if (ret)
  4964. return ret;
  4965. mutex_lock(&memcg->thresholds_lock);
  4966. if (type == _MEM)
  4967. thresholds = &memcg->thresholds;
  4968. else if (type == _MEMSWAP)
  4969. thresholds = &memcg->memsw_thresholds;
  4970. else
  4971. BUG();
  4972. usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
  4973. /* Check if a threshold crossed before adding a new one */
  4974. if (thresholds->primary)
  4975. __mem_cgroup_threshold(memcg, type == _MEMSWAP);
  4976. size = thresholds->primary ? thresholds->primary->size + 1 : 1;
  4977. /* Allocate memory for new array of thresholds */
  4978. new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
  4979. GFP_KERNEL);
  4980. if (!new) {
  4981. ret = -ENOMEM;
  4982. goto unlock;
  4983. }
  4984. new->size = size;
  4985. /* Copy thresholds (if any) to new array */
  4986. if (thresholds->primary) {
  4987. memcpy(new->entries, thresholds->primary->entries, (size - 1) *
  4988. sizeof(struct mem_cgroup_threshold));
  4989. }
  4990. /* Add new threshold */
  4991. new->entries[size - 1].eventfd = eventfd;
  4992. new->entries[size - 1].threshold = threshold;
  4993. /* Sort thresholds. Registering of new threshold isn't time-critical */
  4994. sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
  4995. compare_thresholds, NULL);
  4996. /* Find current threshold */
  4997. new->current_threshold = -1;
  4998. for (i = 0; i < size; i++) {
  4999. if (new->entries[i].threshold <= usage) {
  5000. /*
  5001. * new->current_threshold will not be used until
  5002. * rcu_assign_pointer(), so it's safe to increment
  5003. * it here.
  5004. */
  5005. ++new->current_threshold;
  5006. } else
  5007. break;
  5008. }
  5009. /* Free old spare buffer and save old primary buffer as spare */
  5010. kfree(thresholds->spare);
  5011. thresholds->spare = thresholds->primary;
  5012. rcu_assign_pointer(thresholds->primary, new);
  5013. /* To be sure that nobody uses thresholds */
  5014. synchronize_rcu();
  5015. unlock:
  5016. mutex_unlock(&memcg->thresholds_lock);
  5017. return ret;
  5018. }
  5019. static void mem_cgroup_usage_unregister_event(struct cgroup_subsys_state *css,
  5020. struct cftype *cft, struct eventfd_ctx *eventfd)
  5021. {
  5022. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  5023. struct mem_cgroup_thresholds *thresholds;
  5024. struct mem_cgroup_threshold_ary *new;
  5025. enum res_type type = MEMFILE_TYPE(cft->private);
  5026. u64 usage;
  5027. int i, j, size;
  5028. mutex_lock(&memcg->thresholds_lock);
  5029. if (type == _MEM)
  5030. thresholds = &memcg->thresholds;
  5031. else if (type == _MEMSWAP)
  5032. thresholds = &memcg->memsw_thresholds;
  5033. else
  5034. BUG();
  5035. if (!thresholds->primary)
  5036. goto unlock;
  5037. usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
  5038. /* Check if a threshold crossed before removing */
  5039. __mem_cgroup_threshold(memcg, type == _MEMSWAP);
  5040. /* Calculate new number of threshold */
  5041. size = 0;
  5042. for (i = 0; i < thresholds->primary->size; i++) {
  5043. if (thresholds->primary->entries[i].eventfd != eventfd)
  5044. size++;
  5045. }
  5046. new = thresholds->spare;
  5047. /* Set thresholds array to NULL if we don't have thresholds */
  5048. if (!size) {
  5049. kfree(new);
  5050. new = NULL;
  5051. goto swap_buffers;
  5052. }
  5053. new->size = size;
  5054. /* Copy thresholds and find current threshold */
  5055. new->current_threshold = -1;
  5056. for (i = 0, j = 0; i < thresholds->primary->size; i++) {
  5057. if (thresholds->primary->entries[i].eventfd == eventfd)
  5058. continue;
  5059. new->entries[j] = thresholds->primary->entries[i];
  5060. if (new->entries[j].threshold <= usage) {
  5061. /*
  5062. * new->current_threshold will not be used
  5063. * until rcu_assign_pointer(), so it's safe to increment
  5064. * it here.
  5065. */
  5066. ++new->current_threshold;
  5067. }
  5068. j++;
  5069. }
  5070. swap_buffers:
  5071. /* Swap primary and spare array */
  5072. thresholds->spare = thresholds->primary;
  5073. /* If all events are unregistered, free the spare array */
  5074. if (!new) {
  5075. kfree(thresholds->spare);
  5076. thresholds->spare = NULL;
  5077. }
  5078. rcu_assign_pointer(thresholds->primary, new);
  5079. /* To be sure that nobody uses thresholds */
  5080. synchronize_rcu();
  5081. unlock:
  5082. mutex_unlock(&memcg->thresholds_lock);
  5083. }
  5084. static int mem_cgroup_oom_register_event(struct cgroup_subsys_state *css,
  5085. struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
  5086. {
  5087. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  5088. struct mem_cgroup_eventfd_list *event;
  5089. enum res_type type = MEMFILE_TYPE(cft->private);
  5090. BUG_ON(type != _OOM_TYPE);
  5091. event = kmalloc(sizeof(*event), GFP_KERNEL);
  5092. if (!event)
  5093. return -ENOMEM;
  5094. spin_lock(&memcg_oom_lock);
  5095. event->eventfd = eventfd;
  5096. list_add(&event->list, &memcg->oom_notify);
  5097. /* already in OOM ? */
  5098. if (atomic_read(&memcg->under_oom))
  5099. eventfd_signal(eventfd, 1);
  5100. spin_unlock(&memcg_oom_lock);
  5101. return 0;
  5102. }
  5103. static void mem_cgroup_oom_unregister_event(struct cgroup_subsys_state *css,
  5104. struct cftype *cft, struct eventfd_ctx *eventfd)
  5105. {
  5106. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  5107. struct mem_cgroup_eventfd_list *ev, *tmp;
  5108. enum res_type type = MEMFILE_TYPE(cft->private);
  5109. BUG_ON(type != _OOM_TYPE);
  5110. spin_lock(&memcg_oom_lock);
  5111. list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
  5112. if (ev->eventfd == eventfd) {
  5113. list_del(&ev->list);
  5114. kfree(ev);
  5115. }
  5116. }
  5117. spin_unlock(&memcg_oom_lock);
  5118. }
  5119. static int mem_cgroup_oom_control_read(struct cgroup_subsys_state *css,
  5120. struct cftype *cft, struct cgroup_map_cb *cb)
  5121. {
  5122. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  5123. cb->fill(cb, "oom_kill_disable", memcg->oom_kill_disable);
  5124. if (atomic_read(&memcg->under_oom))
  5125. cb->fill(cb, "under_oom", 1);
  5126. else
  5127. cb->fill(cb, "under_oom", 0);
  5128. return 0;
  5129. }
  5130. static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
  5131. struct cftype *cft, u64 val)
  5132. {
  5133. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  5134. struct mem_cgroup *parent = mem_cgroup_from_css(css_parent(&memcg->css));
  5135. /* cannot set to root cgroup and only 0 and 1 are allowed */
  5136. if (!parent || !((val == 0) || (val == 1)))
  5137. return -EINVAL;
  5138. mutex_lock(&memcg_create_mutex);
  5139. /* oom-kill-disable is a flag for subhierarchy. */
  5140. if ((parent->use_hierarchy) || memcg_has_children(memcg)) {
  5141. mutex_unlock(&memcg_create_mutex);
  5142. return -EINVAL;
  5143. }
  5144. memcg->oom_kill_disable = val;
  5145. if (!val)
  5146. memcg_oom_recover(memcg);
  5147. mutex_unlock(&memcg_create_mutex);
  5148. return 0;
  5149. }
  5150. #ifdef CONFIG_MEMCG_KMEM
  5151. static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
  5152. {
  5153. int ret;
  5154. memcg->kmemcg_id = -1;
  5155. ret = memcg_propagate_kmem(memcg);
  5156. if (ret)
  5157. return ret;
  5158. return mem_cgroup_sockets_init(memcg, ss);
  5159. }
  5160. static void memcg_destroy_kmem(struct mem_cgroup *memcg)
  5161. {
  5162. mem_cgroup_sockets_destroy(memcg);
  5163. }
  5164. static void kmem_cgroup_css_offline(struct mem_cgroup *memcg)
  5165. {
  5166. if (!memcg_kmem_is_active(memcg))
  5167. return;
  5168. /*
  5169. * kmem charges can outlive the cgroup. In the case of slab
  5170. * pages, for instance, a page contain objects from various
  5171. * processes. As we prevent from taking a reference for every
  5172. * such allocation we have to be careful when doing uncharge
  5173. * (see memcg_uncharge_kmem) and here during offlining.
  5174. *
  5175. * The idea is that that only the _last_ uncharge which sees
  5176. * the dead memcg will drop the last reference. An additional
  5177. * reference is taken here before the group is marked dead
  5178. * which is then paired with css_put during uncharge resp. here.
  5179. *
  5180. * Although this might sound strange as this path is called from
  5181. * css_offline() when the referencemight have dropped down to 0
  5182. * and shouldn't be incremented anymore (css_tryget would fail)
  5183. * we do not have other options because of the kmem allocations
  5184. * lifetime.
  5185. */
  5186. css_get(&memcg->css);
  5187. memcg_kmem_mark_dead(memcg);
  5188. if (res_counter_read_u64(&memcg->kmem, RES_USAGE) != 0)
  5189. return;
  5190. if (memcg_kmem_test_and_clear_dead(memcg))
  5191. css_put(&memcg->css);
  5192. }
  5193. #else
  5194. static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
  5195. {
  5196. return 0;
  5197. }
  5198. static void memcg_destroy_kmem(struct mem_cgroup *memcg)
  5199. {
  5200. }
  5201. static void kmem_cgroup_css_offline(struct mem_cgroup *memcg)
  5202. {
  5203. }
  5204. #endif
  5205. static struct cftype mem_cgroup_files[] = {
  5206. {
  5207. .name = "usage_in_bytes",
  5208. .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
  5209. .read = mem_cgroup_read,
  5210. .register_event = mem_cgroup_usage_register_event,
  5211. .unregister_event = mem_cgroup_usage_unregister_event,
  5212. },
  5213. {
  5214. .name = "max_usage_in_bytes",
  5215. .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
  5216. .trigger = mem_cgroup_reset,
  5217. .read = mem_cgroup_read,
  5218. },
  5219. {
  5220. .name = "limit_in_bytes",
  5221. .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
  5222. .write_string = mem_cgroup_write,
  5223. .read = mem_cgroup_read,
  5224. },
  5225. {
  5226. .name = "soft_limit_in_bytes",
  5227. .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
  5228. .write_string = mem_cgroup_write,
  5229. .read = mem_cgroup_read,
  5230. },
  5231. {
  5232. .name = "failcnt",
  5233. .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
  5234. .trigger = mem_cgroup_reset,
  5235. .read = mem_cgroup_read,
  5236. },
  5237. {
  5238. .name = "stat",
  5239. .read_seq_string = memcg_stat_show,
  5240. },
  5241. {
  5242. .name = "force_empty",
  5243. .trigger = mem_cgroup_force_empty_write,
  5244. },
  5245. {
  5246. .name = "use_hierarchy",
  5247. .flags = CFTYPE_INSANE,
  5248. .write_u64 = mem_cgroup_hierarchy_write,
  5249. .read_u64 = mem_cgroup_hierarchy_read,
  5250. },
  5251. {
  5252. .name = "swappiness",
  5253. .read_u64 = mem_cgroup_swappiness_read,
  5254. .write_u64 = mem_cgroup_swappiness_write,
  5255. },
  5256. {
  5257. .name = "move_charge_at_immigrate",
  5258. .read_u64 = mem_cgroup_move_charge_read,
  5259. .write_u64 = mem_cgroup_move_charge_write,
  5260. },
  5261. {
  5262. .name = "oom_control",
  5263. .read_map = mem_cgroup_oom_control_read,
  5264. .write_u64 = mem_cgroup_oom_control_write,
  5265. .register_event = mem_cgroup_oom_register_event,
  5266. .unregister_event = mem_cgroup_oom_unregister_event,
  5267. .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
  5268. },
  5269. {
  5270. .name = "pressure_level",
  5271. .register_event = vmpressure_register_event,
  5272. .unregister_event = vmpressure_unregister_event,
  5273. },
  5274. #ifdef CONFIG_NUMA
  5275. {
  5276. .name = "numa_stat",
  5277. .read_seq_string = memcg_numa_stat_show,
  5278. },
  5279. #endif
  5280. #ifdef CONFIG_MEMCG_KMEM
  5281. {
  5282. .name = "kmem.limit_in_bytes",
  5283. .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
  5284. .write_string = mem_cgroup_write,
  5285. .read = mem_cgroup_read,
  5286. },
  5287. {
  5288. .name = "kmem.usage_in_bytes",
  5289. .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
  5290. .read = mem_cgroup_read,
  5291. },
  5292. {
  5293. .name = "kmem.failcnt",
  5294. .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
  5295. .trigger = mem_cgroup_reset,
  5296. .read = mem_cgroup_read,
  5297. },
  5298. {
  5299. .name = "kmem.max_usage_in_bytes",
  5300. .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
  5301. .trigger = mem_cgroup_reset,
  5302. .read = mem_cgroup_read,
  5303. },
  5304. #ifdef CONFIG_SLABINFO
  5305. {
  5306. .name = "kmem.slabinfo",
  5307. .read_seq_string = mem_cgroup_slabinfo_read,
  5308. },
  5309. #endif
  5310. #endif
  5311. { }, /* terminate */
  5312. };
  5313. #ifdef CONFIG_MEMCG_SWAP
  5314. static struct cftype memsw_cgroup_files[] = {
  5315. {
  5316. .name = "memsw.usage_in_bytes",
  5317. .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
  5318. .read = mem_cgroup_read,
  5319. .register_event = mem_cgroup_usage_register_event,
  5320. .unregister_event = mem_cgroup_usage_unregister_event,
  5321. },
  5322. {
  5323. .name = "memsw.max_usage_in_bytes",
  5324. .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
  5325. .trigger = mem_cgroup_reset,
  5326. .read = mem_cgroup_read,
  5327. },
  5328. {
  5329. .name = "memsw.limit_in_bytes",
  5330. .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
  5331. .write_string = mem_cgroup_write,
  5332. .read = mem_cgroup_read,
  5333. },
  5334. {
  5335. .name = "memsw.failcnt",
  5336. .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
  5337. .trigger = mem_cgroup_reset,
  5338. .read = mem_cgroup_read,
  5339. },
  5340. { }, /* terminate */
  5341. };
  5342. #endif
  5343. static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
  5344. {
  5345. struct mem_cgroup_per_node *pn;
  5346. struct mem_cgroup_per_zone *mz;
  5347. int zone, tmp = node;
  5348. /*
  5349. * This routine is called against possible nodes.
  5350. * But it's BUG to call kmalloc() against offline node.
  5351. *
  5352. * TODO: this routine can waste much memory for nodes which will
  5353. * never be onlined. It's better to use memory hotplug callback
  5354. * function.
  5355. */
  5356. if (!node_state(node, N_NORMAL_MEMORY))
  5357. tmp = -1;
  5358. pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
  5359. if (!pn)
  5360. return 1;
  5361. for (zone = 0; zone < MAX_NR_ZONES; zone++) {
  5362. mz = &pn->zoneinfo[zone];
  5363. lruvec_init(&mz->lruvec);
  5364. mz->usage_in_excess = 0;
  5365. mz->on_tree = false;
  5366. mz->memcg = memcg;
  5367. }
  5368. memcg->nodeinfo[node] = pn;
  5369. return 0;
  5370. }
  5371. static void free_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
  5372. {
  5373. kfree(memcg->nodeinfo[node]);
  5374. }
  5375. static struct mem_cgroup *mem_cgroup_alloc(void)
  5376. {
  5377. struct mem_cgroup *memcg;
  5378. size_t size = memcg_size();
  5379. /* Can be very big if nr_node_ids is very big */
  5380. if (size < PAGE_SIZE)
  5381. memcg = kzalloc(size, GFP_KERNEL);
  5382. else
  5383. memcg = vzalloc(size);
  5384. if (!memcg)
  5385. return NULL;
  5386. memcg->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
  5387. if (!memcg->stat)
  5388. goto out_free;
  5389. spin_lock_init(&memcg->pcp_counter_lock);
  5390. return memcg;
  5391. out_free:
  5392. if (size < PAGE_SIZE)
  5393. kfree(memcg);
  5394. else
  5395. vfree(memcg);
  5396. return NULL;
  5397. }
  5398. /*
  5399. * At destroying mem_cgroup, references from swap_cgroup can remain.
  5400. * (scanning all at force_empty is too costly...)
  5401. *
  5402. * Instead of clearing all references at force_empty, we remember
  5403. * the number of reference from swap_cgroup and free mem_cgroup when
  5404. * it goes down to 0.
  5405. *
  5406. * Removal of cgroup itself succeeds regardless of refs from swap.
  5407. */
  5408. static void __mem_cgroup_free(struct mem_cgroup *memcg)
  5409. {
  5410. int node;
  5411. size_t size = memcg_size();
  5412. mem_cgroup_remove_from_trees(memcg);
  5413. for_each_node(node)
  5414. free_mem_cgroup_per_zone_info(memcg, node);
  5415. free_percpu(memcg->stat);
  5416. /*
  5417. * We need to make sure that (at least for now), the jump label
  5418. * destruction code runs outside of the cgroup lock. This is because
  5419. * get_online_cpus(), which is called from the static_branch update,
  5420. * can't be called inside the cgroup_lock. cpusets are the ones
  5421. * enforcing this dependency, so if they ever change, we might as well.
  5422. *
  5423. * schedule_work() will guarantee this happens. Be careful if you need
  5424. * to move this code around, and make sure it is outside
  5425. * the cgroup_lock.
  5426. */
  5427. disarm_static_keys(memcg);
  5428. if (size < PAGE_SIZE)
  5429. kfree(memcg);
  5430. else
  5431. vfree(memcg);
  5432. }
  5433. /*
  5434. * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
  5435. */
  5436. struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
  5437. {
  5438. if (!memcg->res.parent)
  5439. return NULL;
  5440. return mem_cgroup_from_res_counter(memcg->res.parent, res);
  5441. }
  5442. EXPORT_SYMBOL(parent_mem_cgroup);
  5443. static void __init mem_cgroup_soft_limit_tree_init(void)
  5444. {
  5445. struct mem_cgroup_tree_per_node *rtpn;
  5446. struct mem_cgroup_tree_per_zone *rtpz;
  5447. int tmp, node, zone;
  5448. for_each_node(node) {
  5449. tmp = node;
  5450. if (!node_state(node, N_NORMAL_MEMORY))
  5451. tmp = -1;
  5452. rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp);
  5453. BUG_ON(!rtpn);
  5454. soft_limit_tree.rb_tree_per_node[node] = rtpn;
  5455. for (zone = 0; zone < MAX_NR_ZONES; zone++) {
  5456. rtpz = &rtpn->rb_tree_per_zone[zone];
  5457. rtpz->rb_root = RB_ROOT;
  5458. spin_lock_init(&rtpz->lock);
  5459. }
  5460. }
  5461. }
  5462. static struct cgroup_subsys_state * __ref
  5463. mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
  5464. {
  5465. struct mem_cgroup *memcg;
  5466. long error = -ENOMEM;
  5467. int node;
  5468. memcg = mem_cgroup_alloc();
  5469. if (!memcg)
  5470. return ERR_PTR(error);
  5471. for_each_node(node)
  5472. if (alloc_mem_cgroup_per_zone_info(memcg, node))
  5473. goto free_out;
  5474. /* root ? */
  5475. if (parent_css == NULL) {
  5476. root_mem_cgroup = memcg;
  5477. res_counter_init(&memcg->res, NULL);
  5478. res_counter_init(&memcg->memsw, NULL);
  5479. res_counter_init(&memcg->kmem, NULL);
  5480. }
  5481. memcg->last_scanned_node = MAX_NUMNODES;
  5482. INIT_LIST_HEAD(&memcg->oom_notify);
  5483. memcg->move_charge_at_immigrate = 0;
  5484. mutex_init(&memcg->thresholds_lock);
  5485. spin_lock_init(&memcg->move_lock);
  5486. vmpressure_init(&memcg->vmpressure);
  5487. return &memcg->css;
  5488. free_out:
  5489. __mem_cgroup_free(memcg);
  5490. return ERR_PTR(error);
  5491. }
  5492. static int
  5493. mem_cgroup_css_online(struct cgroup_subsys_state *css)
  5494. {
  5495. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  5496. struct mem_cgroup *parent = mem_cgroup_from_css(css_parent(css));
  5497. int error = 0;
  5498. if (css->cgroup->id > MEM_CGROUP_ID_MAX)
  5499. return -ENOSPC;
  5500. if (!parent)
  5501. return 0;
  5502. mutex_lock(&memcg_create_mutex);
  5503. memcg->use_hierarchy = parent->use_hierarchy;
  5504. memcg->oom_kill_disable = parent->oom_kill_disable;
  5505. memcg->swappiness = mem_cgroup_swappiness(parent);
  5506. if (parent->use_hierarchy) {
  5507. res_counter_init(&memcg->res, &parent->res);
  5508. res_counter_init(&memcg->memsw, &parent->memsw);
  5509. res_counter_init(&memcg->kmem, &parent->kmem);
  5510. /*
  5511. * No need to take a reference to the parent because cgroup
  5512. * core guarantees its existence.
  5513. */
  5514. } else {
  5515. res_counter_init(&memcg->res, NULL);
  5516. res_counter_init(&memcg->memsw, NULL);
  5517. res_counter_init(&memcg->kmem, NULL);
  5518. /*
  5519. * Deeper hierachy with use_hierarchy == false doesn't make
  5520. * much sense so let cgroup subsystem know about this
  5521. * unfortunate state in our controller.
  5522. */
  5523. if (parent != root_mem_cgroup)
  5524. mem_cgroup_subsys.broken_hierarchy = true;
  5525. }
  5526. error = memcg_init_kmem(memcg, &mem_cgroup_subsys);
  5527. mutex_unlock(&memcg_create_mutex);
  5528. return error;
  5529. }
  5530. /*
  5531. * Announce all parents that a group from their hierarchy is gone.
  5532. */
  5533. static void mem_cgroup_invalidate_reclaim_iterators(struct mem_cgroup *memcg)
  5534. {
  5535. struct mem_cgroup *parent = memcg;
  5536. while ((parent = parent_mem_cgroup(parent)))
  5537. mem_cgroup_iter_invalidate(parent);
  5538. /*
  5539. * if the root memcg is not hierarchical we have to check it
  5540. * explicitely.
  5541. */
  5542. if (!root_mem_cgroup->use_hierarchy)
  5543. mem_cgroup_iter_invalidate(root_mem_cgroup);
  5544. }
  5545. static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
  5546. {
  5547. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  5548. kmem_cgroup_css_offline(memcg);
  5549. mem_cgroup_invalidate_reclaim_iterators(memcg);
  5550. mem_cgroup_reparent_charges(memcg);
  5551. mem_cgroup_destroy_all_caches(memcg);
  5552. vmpressure_cleanup(&memcg->vmpressure);
  5553. }
  5554. static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
  5555. {
  5556. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  5557. memcg_destroy_kmem(memcg);
  5558. __mem_cgroup_free(memcg);
  5559. }
  5560. #ifdef CONFIG_MMU
  5561. /* Handlers for move charge at task migration. */
  5562. #define PRECHARGE_COUNT_AT_ONCE 256
  5563. static int mem_cgroup_do_precharge(unsigned long count)
  5564. {
  5565. int ret = 0;
  5566. int batch_count = PRECHARGE_COUNT_AT_ONCE;
  5567. struct mem_cgroup *memcg = mc.to;
  5568. if (mem_cgroup_is_root(memcg)) {
  5569. mc.precharge += count;
  5570. /* we don't need css_get for root */
  5571. return ret;
  5572. }
  5573. /* try to charge at once */
  5574. if (count > 1) {
  5575. struct res_counter *dummy;
  5576. /*
  5577. * "memcg" cannot be under rmdir() because we've already checked
  5578. * by cgroup_lock_live_cgroup() that it is not removed and we
  5579. * are still under the same cgroup_mutex. So we can postpone
  5580. * css_get().
  5581. */
  5582. if (res_counter_charge(&memcg->res, PAGE_SIZE * count, &dummy))
  5583. goto one_by_one;
  5584. if (do_swap_account && res_counter_charge(&memcg->memsw,
  5585. PAGE_SIZE * count, &dummy)) {
  5586. res_counter_uncharge(&memcg->res, PAGE_SIZE * count);
  5587. goto one_by_one;
  5588. }
  5589. mc.precharge += count;
  5590. return ret;
  5591. }
  5592. one_by_one:
  5593. /* fall back to one by one charge */
  5594. while (count--) {
  5595. if (signal_pending(current)) {
  5596. ret = -EINTR;
  5597. break;
  5598. }
  5599. if (!batch_count--) {
  5600. batch_count = PRECHARGE_COUNT_AT_ONCE;
  5601. cond_resched();
  5602. }
  5603. ret = __mem_cgroup_try_charge(NULL,
  5604. GFP_KERNEL, 1, &memcg, false);
  5605. if (ret)
  5606. /* mem_cgroup_clear_mc() will do uncharge later */
  5607. return ret;
  5608. mc.precharge++;
  5609. }
  5610. return ret;
  5611. }
  5612. /**
  5613. * get_mctgt_type - get target type of moving charge
  5614. * @vma: the vma the pte to be checked belongs
  5615. * @addr: the address corresponding to the pte to be checked
  5616. * @ptent: the pte to be checked
  5617. * @target: the pointer the target page or swap ent will be stored(can be NULL)
  5618. *
  5619. * Returns
  5620. * 0(MC_TARGET_NONE): if the pte is not a target for move charge.
  5621. * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
  5622. * move charge. if @target is not NULL, the page is stored in target->page
  5623. * with extra refcnt got(Callers should handle it).
  5624. * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
  5625. * target for charge migration. if @target is not NULL, the entry is stored
  5626. * in target->ent.
  5627. *
  5628. * Called with pte lock held.
  5629. */
  5630. union mc_target {
  5631. struct page *page;
  5632. swp_entry_t ent;
  5633. };
  5634. enum mc_target_type {
  5635. MC_TARGET_NONE = 0,
  5636. MC_TARGET_PAGE,
  5637. MC_TARGET_SWAP,
  5638. };
  5639. static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
  5640. unsigned long addr, pte_t ptent)
  5641. {
  5642. struct page *page = vm_normal_page(vma, addr, ptent);
  5643. if (!page || !page_mapped(page))
  5644. return NULL;
  5645. if (PageAnon(page)) {
  5646. /* we don't move shared anon */
  5647. if (!move_anon())
  5648. return NULL;
  5649. } else if (!move_file())
  5650. /* we ignore mapcount for file pages */
  5651. return NULL;
  5652. if (!get_page_unless_zero(page))
  5653. return NULL;
  5654. return page;
  5655. }
  5656. #ifdef CONFIG_SWAP
  5657. static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
  5658. unsigned long addr, pte_t ptent, swp_entry_t *entry)
  5659. {
  5660. struct page *page = NULL;
  5661. swp_entry_t ent = pte_to_swp_entry(ptent);
  5662. if (!move_anon() || non_swap_entry(ent))
  5663. return NULL;
  5664. /*
  5665. * Because lookup_swap_cache() updates some statistics counter,
  5666. * we call find_get_page() with swapper_space directly.
  5667. */
  5668. page = find_get_page(swap_address_space(ent), ent.val);
  5669. if (do_swap_account)
  5670. entry->val = ent.val;
  5671. return page;
  5672. }
  5673. #else
  5674. static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
  5675. unsigned long addr, pte_t ptent, swp_entry_t *entry)
  5676. {
  5677. return NULL;
  5678. }
  5679. #endif
  5680. static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
  5681. unsigned long addr, pte_t ptent, swp_entry_t *entry)
  5682. {
  5683. struct page *page = NULL;
  5684. struct address_space *mapping;
  5685. pgoff_t pgoff;
  5686. if (!vma->vm_file) /* anonymous vma */
  5687. return NULL;
  5688. if (!move_file())
  5689. return NULL;
  5690. mapping = vma->vm_file->f_mapping;
  5691. if (pte_none(ptent))
  5692. pgoff = linear_page_index(vma, addr);
  5693. else /* pte_file(ptent) is true */
  5694. pgoff = pte_to_pgoff(ptent);
  5695. /* page is moved even if it's not RSS of this task(page-faulted). */
  5696. page = find_get_page(mapping, pgoff);
  5697. #ifdef CONFIG_SWAP
  5698. /* shmem/tmpfs may report page out on swap: account for that too. */
  5699. if (radix_tree_exceptional_entry(page)) {
  5700. swp_entry_t swap = radix_to_swp_entry(page);
  5701. if (do_swap_account)
  5702. *entry = swap;
  5703. page = find_get_page(swap_address_space(swap), swap.val);
  5704. }
  5705. #endif
  5706. return page;
  5707. }
  5708. static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
  5709. unsigned long addr, pte_t ptent, union mc_target *target)
  5710. {
  5711. struct page *page = NULL;
  5712. struct page_cgroup *pc;
  5713. enum mc_target_type ret = MC_TARGET_NONE;
  5714. swp_entry_t ent = { .val = 0 };
  5715. if (pte_present(ptent))
  5716. page = mc_handle_present_pte(vma, addr, ptent);
  5717. else if (is_swap_pte(ptent))
  5718. page = mc_handle_swap_pte(vma, addr, ptent, &ent);
  5719. else if (pte_none(ptent) || pte_file(ptent))
  5720. page = mc_handle_file_pte(vma, addr, ptent, &ent);
  5721. if (!page && !ent.val)
  5722. return ret;
  5723. if (page) {
  5724. pc = lookup_page_cgroup(page);
  5725. /*
  5726. * Do only loose check w/o page_cgroup lock.
  5727. * mem_cgroup_move_account() checks the pc is valid or not under
  5728. * the lock.
  5729. */
  5730. if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
  5731. ret = MC_TARGET_PAGE;
  5732. if (target)
  5733. target->page = page;
  5734. }
  5735. if (!ret || !target)
  5736. put_page(page);
  5737. }
  5738. /* There is a swap entry and a page doesn't exist or isn't charged */
  5739. if (ent.val && !ret &&
  5740. mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
  5741. ret = MC_TARGET_SWAP;
  5742. if (target)
  5743. target->ent = ent;
  5744. }
  5745. return ret;
  5746. }
  5747. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  5748. /*
  5749. * We don't consider swapping or file mapped pages because THP does not
  5750. * support them for now.
  5751. * Caller should make sure that pmd_trans_huge(pmd) is true.
  5752. */
  5753. static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
  5754. unsigned long addr, pmd_t pmd, union mc_target *target)
  5755. {
  5756. struct page *page = NULL;
  5757. struct page_cgroup *pc;
  5758. enum mc_target_type ret = MC_TARGET_NONE;
  5759. page = pmd_page(pmd);
  5760. VM_BUG_ON(!page || !PageHead(page));
  5761. if (!move_anon())
  5762. return ret;
  5763. pc = lookup_page_cgroup(page);
  5764. if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
  5765. ret = MC_TARGET_PAGE;
  5766. if (target) {
  5767. get_page(page);
  5768. target->page = page;
  5769. }
  5770. }
  5771. return ret;
  5772. }
  5773. #else
  5774. static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
  5775. unsigned long addr, pmd_t pmd, union mc_target *target)
  5776. {
  5777. return MC_TARGET_NONE;
  5778. }
  5779. #endif
  5780. static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
  5781. unsigned long addr, unsigned long end,
  5782. struct mm_walk *walk)
  5783. {
  5784. struct vm_area_struct *vma = walk->private;
  5785. pte_t *pte;
  5786. spinlock_t *ptl;
  5787. if (pmd_trans_huge_lock(pmd, vma, &ptl) == 1) {
  5788. if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
  5789. mc.precharge += HPAGE_PMD_NR;
  5790. spin_unlock(ptl);
  5791. return 0;
  5792. }
  5793. if (pmd_trans_unstable(pmd))
  5794. return 0;
  5795. pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
  5796. for (; addr != end; pte++, addr += PAGE_SIZE)
  5797. if (get_mctgt_type(vma, addr, *pte, NULL))
  5798. mc.precharge++; /* increment precharge temporarily */
  5799. pte_unmap_unlock(pte - 1, ptl);
  5800. cond_resched();
  5801. return 0;
  5802. }
  5803. static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
  5804. {
  5805. unsigned long precharge;
  5806. struct vm_area_struct *vma;
  5807. down_read(&mm->mmap_sem);
  5808. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  5809. struct mm_walk mem_cgroup_count_precharge_walk = {
  5810. .pmd_entry = mem_cgroup_count_precharge_pte_range,
  5811. .mm = mm,
  5812. .private = vma,
  5813. };
  5814. if (is_vm_hugetlb_page(vma))
  5815. continue;
  5816. walk_page_range(vma->vm_start, vma->vm_end,
  5817. &mem_cgroup_count_precharge_walk);
  5818. }
  5819. up_read(&mm->mmap_sem);
  5820. precharge = mc.precharge;
  5821. mc.precharge = 0;
  5822. return precharge;
  5823. }
  5824. static int mem_cgroup_precharge_mc(struct mm_struct *mm)
  5825. {
  5826. unsigned long precharge = mem_cgroup_count_precharge(mm);
  5827. VM_BUG_ON(mc.moving_task);
  5828. mc.moving_task = current;
  5829. return mem_cgroup_do_precharge(precharge);
  5830. }
  5831. /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
  5832. static void __mem_cgroup_clear_mc(void)
  5833. {
  5834. struct mem_cgroup *from = mc.from;
  5835. struct mem_cgroup *to = mc.to;
  5836. int i;
  5837. /* we must uncharge all the leftover precharges from mc.to */
  5838. if (mc.precharge) {
  5839. __mem_cgroup_cancel_charge(mc.to, mc.precharge);
  5840. mc.precharge = 0;
  5841. }
  5842. /*
  5843. * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
  5844. * we must uncharge here.
  5845. */
  5846. if (mc.moved_charge) {
  5847. __mem_cgroup_cancel_charge(mc.from, mc.moved_charge);
  5848. mc.moved_charge = 0;
  5849. }
  5850. /* we must fixup refcnts and charges */
  5851. if (mc.moved_swap) {
  5852. /* uncharge swap account from the old cgroup */
  5853. if (!mem_cgroup_is_root(mc.from))
  5854. res_counter_uncharge(&mc.from->memsw,
  5855. PAGE_SIZE * mc.moved_swap);
  5856. for (i = 0; i < mc.moved_swap; i++)
  5857. css_put(&mc.from->css);
  5858. if (!mem_cgroup_is_root(mc.to)) {
  5859. /*
  5860. * we charged both to->res and to->memsw, so we should
  5861. * uncharge to->res.
  5862. */
  5863. res_counter_uncharge(&mc.to->res,
  5864. PAGE_SIZE * mc.moved_swap);
  5865. }
  5866. /* we've already done css_get(mc.to) */
  5867. mc.moved_swap = 0;
  5868. }
  5869. memcg_oom_recover(from);
  5870. memcg_oom_recover(to);
  5871. wake_up_all(&mc.waitq);
  5872. }
  5873. static void mem_cgroup_clear_mc(void)
  5874. {
  5875. struct mem_cgroup *from = mc.from;
  5876. /*
  5877. * we must clear moving_task before waking up waiters at the end of
  5878. * task migration.
  5879. */
  5880. mc.moving_task = NULL;
  5881. __mem_cgroup_clear_mc();
  5882. spin_lock(&mc.lock);
  5883. mc.from = NULL;
  5884. mc.to = NULL;
  5885. spin_unlock(&mc.lock);
  5886. mem_cgroup_end_move(from);
  5887. }
  5888. static int mem_cgroup_can_attach(struct cgroup_subsys_state *css,
  5889. struct cgroup_taskset *tset)
  5890. {
  5891. struct task_struct *p = cgroup_taskset_first(tset);
  5892. int ret = 0;
  5893. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  5894. unsigned long move_charge_at_immigrate;
  5895. /*
  5896. * We are now commited to this value whatever it is. Changes in this
  5897. * tunable will only affect upcoming migrations, not the current one.
  5898. * So we need to save it, and keep it going.
  5899. */
  5900. move_charge_at_immigrate = memcg->move_charge_at_immigrate;
  5901. if (move_charge_at_immigrate) {
  5902. struct mm_struct *mm;
  5903. struct mem_cgroup *from = mem_cgroup_from_task(p);
  5904. VM_BUG_ON(from == memcg);
  5905. mm = get_task_mm(p);
  5906. if (!mm)
  5907. return 0;
  5908. /* We move charges only when we move a owner of the mm */
  5909. if (mm->owner == p) {
  5910. VM_BUG_ON(mc.from);
  5911. VM_BUG_ON(mc.to);
  5912. VM_BUG_ON(mc.precharge);
  5913. VM_BUG_ON(mc.moved_charge);
  5914. VM_BUG_ON(mc.moved_swap);
  5915. mem_cgroup_start_move(from);
  5916. spin_lock(&mc.lock);
  5917. mc.from = from;
  5918. mc.to = memcg;
  5919. mc.immigrate_flags = move_charge_at_immigrate;
  5920. spin_unlock(&mc.lock);
  5921. /* We set mc.moving_task later */
  5922. ret = mem_cgroup_precharge_mc(mm);
  5923. if (ret)
  5924. mem_cgroup_clear_mc();
  5925. }
  5926. mmput(mm);
  5927. }
  5928. return ret;
  5929. }
  5930. static void mem_cgroup_cancel_attach(struct cgroup_subsys_state *css,
  5931. struct cgroup_taskset *tset)
  5932. {
  5933. mem_cgroup_clear_mc();
  5934. }
  5935. static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
  5936. unsigned long addr, unsigned long end,
  5937. struct mm_walk *walk)
  5938. {
  5939. int ret = 0;
  5940. struct vm_area_struct *vma = walk->private;
  5941. pte_t *pte;
  5942. spinlock_t *ptl;
  5943. enum mc_target_type target_type;
  5944. union mc_target target;
  5945. struct page *page;
  5946. struct page_cgroup *pc;
  5947. /*
  5948. * We don't take compound_lock() here but no race with splitting thp
  5949. * happens because:
  5950. * - if pmd_trans_huge_lock() returns 1, the relevant thp is not
  5951. * under splitting, which means there's no concurrent thp split,
  5952. * - if another thread runs into split_huge_page() just after we
  5953. * entered this if-block, the thread must wait for page table lock
  5954. * to be unlocked in __split_huge_page_splitting(), where the main
  5955. * part of thp split is not executed yet.
  5956. */
  5957. if (pmd_trans_huge_lock(pmd, vma, &ptl) == 1) {
  5958. if (mc.precharge < HPAGE_PMD_NR) {
  5959. spin_unlock(ptl);
  5960. return 0;
  5961. }
  5962. target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
  5963. if (target_type == MC_TARGET_PAGE) {
  5964. page = target.page;
  5965. if (!isolate_lru_page(page)) {
  5966. pc = lookup_page_cgroup(page);
  5967. if (!mem_cgroup_move_account(page, HPAGE_PMD_NR,
  5968. pc, mc.from, mc.to)) {
  5969. mc.precharge -= HPAGE_PMD_NR;
  5970. mc.moved_charge += HPAGE_PMD_NR;
  5971. }
  5972. putback_lru_page(page);
  5973. }
  5974. put_page(page);
  5975. }
  5976. spin_unlock(ptl);
  5977. return 0;
  5978. }
  5979. if (pmd_trans_unstable(pmd))
  5980. return 0;
  5981. retry:
  5982. pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
  5983. for (; addr != end; addr += PAGE_SIZE) {
  5984. pte_t ptent = *(pte++);
  5985. swp_entry_t ent;
  5986. if (!mc.precharge)
  5987. break;
  5988. switch (get_mctgt_type(vma, addr, ptent, &target)) {
  5989. case MC_TARGET_PAGE:
  5990. page = target.page;
  5991. if (isolate_lru_page(page))
  5992. goto put;
  5993. pc = lookup_page_cgroup(page);
  5994. if (!mem_cgroup_move_account(page, 1, pc,
  5995. mc.from, mc.to)) {
  5996. mc.precharge--;
  5997. /* we uncharge from mc.from later. */
  5998. mc.moved_charge++;
  5999. }
  6000. putback_lru_page(page);
  6001. put: /* get_mctgt_type() gets the page */
  6002. put_page(page);
  6003. break;
  6004. case MC_TARGET_SWAP:
  6005. ent = target.ent;
  6006. if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
  6007. mc.precharge--;
  6008. /* we fixup refcnts and charges later. */
  6009. mc.moved_swap++;
  6010. }
  6011. break;
  6012. default:
  6013. break;
  6014. }
  6015. }
  6016. pte_unmap_unlock(pte - 1, ptl);
  6017. cond_resched();
  6018. if (addr != end) {
  6019. /*
  6020. * We have consumed all precharges we got in can_attach().
  6021. * We try charge one by one, but don't do any additional
  6022. * charges to mc.to if we have failed in charge once in attach()
  6023. * phase.
  6024. */
  6025. ret = mem_cgroup_do_precharge(1);
  6026. if (!ret)
  6027. goto retry;
  6028. }
  6029. return ret;
  6030. }
  6031. static void mem_cgroup_move_charge(struct mm_struct *mm)
  6032. {
  6033. struct vm_area_struct *vma;
  6034. lru_add_drain_all();
  6035. retry:
  6036. if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
  6037. /*
  6038. * Someone who are holding the mmap_sem might be waiting in
  6039. * waitq. So we cancel all extra charges, wake up all waiters,
  6040. * and retry. Because we cancel precharges, we might not be able
  6041. * to move enough charges, but moving charge is a best-effort
  6042. * feature anyway, so it wouldn't be a big problem.
  6043. */
  6044. __mem_cgroup_clear_mc();
  6045. cond_resched();
  6046. goto retry;
  6047. }
  6048. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  6049. int ret;
  6050. struct mm_walk mem_cgroup_move_charge_walk = {
  6051. .pmd_entry = mem_cgroup_move_charge_pte_range,
  6052. .mm = mm,
  6053. .private = vma,
  6054. };
  6055. if (is_vm_hugetlb_page(vma))
  6056. continue;
  6057. ret = walk_page_range(vma->vm_start, vma->vm_end,
  6058. &mem_cgroup_move_charge_walk);
  6059. if (ret)
  6060. /*
  6061. * means we have consumed all precharges and failed in
  6062. * doing additional charge. Just abandon here.
  6063. */
  6064. break;
  6065. }
  6066. up_read(&mm->mmap_sem);
  6067. }
  6068. static void mem_cgroup_move_task(struct cgroup_subsys_state *css,
  6069. struct cgroup_taskset *tset)
  6070. {
  6071. struct task_struct *p = cgroup_taskset_first(tset);
  6072. struct mm_struct *mm = get_task_mm(p);
  6073. if (mm) {
  6074. if (mc.to)
  6075. mem_cgroup_move_charge(mm);
  6076. mmput(mm);
  6077. }
  6078. if (mc.to)
  6079. mem_cgroup_clear_mc();
  6080. }
  6081. #else /* !CONFIG_MMU */
  6082. static int mem_cgroup_can_attach(struct cgroup_subsys_state *css,
  6083. struct cgroup_taskset *tset)
  6084. {
  6085. return 0;
  6086. }
  6087. static void mem_cgroup_cancel_attach(struct cgroup_subsys_state *css,
  6088. struct cgroup_taskset *tset)
  6089. {
  6090. }
  6091. static void mem_cgroup_move_task(struct cgroup_subsys_state *css,
  6092. struct cgroup_taskset *tset)
  6093. {
  6094. }
  6095. #endif
  6096. /*
  6097. * Cgroup retains root cgroups across [un]mount cycles making it necessary
  6098. * to verify sane_behavior flag on each mount attempt.
  6099. */
  6100. static void mem_cgroup_bind(struct cgroup_subsys_state *root_css)
  6101. {
  6102. /*
  6103. * use_hierarchy is forced with sane_behavior. cgroup core
  6104. * guarantees that @root doesn't have any children, so turning it
  6105. * on for the root memcg is enough.
  6106. */
  6107. if (cgroup_sane_behavior(root_css->cgroup))
  6108. mem_cgroup_from_css(root_css)->use_hierarchy = true;
  6109. }
  6110. struct cgroup_subsys mem_cgroup_subsys = {
  6111. .name = "memory",
  6112. .subsys_id = mem_cgroup_subsys_id,
  6113. .css_alloc = mem_cgroup_css_alloc,
  6114. .css_online = mem_cgroup_css_online,
  6115. .css_offline = mem_cgroup_css_offline,
  6116. .css_free = mem_cgroup_css_free,
  6117. .can_attach = mem_cgroup_can_attach,
  6118. .cancel_attach = mem_cgroup_cancel_attach,
  6119. .attach = mem_cgroup_move_task,
  6120. .bind = mem_cgroup_bind,
  6121. .base_cftypes = mem_cgroup_files,
  6122. .early_init = 0,
  6123. };
  6124. #ifdef CONFIG_MEMCG_SWAP
  6125. static int __init enable_swap_account(char *s)
  6126. {
  6127. if (!strcmp(s, "1"))
  6128. really_do_swap_account = 1;
  6129. else if (!strcmp(s, "0"))
  6130. really_do_swap_account = 0;
  6131. return 1;
  6132. }
  6133. __setup("swapaccount=", enable_swap_account);
  6134. static void __init memsw_file_init(void)
  6135. {
  6136. WARN_ON(cgroup_add_cftypes(&mem_cgroup_subsys, memsw_cgroup_files));
  6137. }
  6138. static void __init enable_swap_cgroup(void)
  6139. {
  6140. if (!mem_cgroup_disabled() && really_do_swap_account) {
  6141. do_swap_account = 1;
  6142. memsw_file_init();
  6143. }
  6144. }
  6145. #else
  6146. static void __init enable_swap_cgroup(void)
  6147. {
  6148. }
  6149. #endif
  6150. /*
  6151. * subsys_initcall() for memory controller.
  6152. *
  6153. * Some parts like hotcpu_notifier() have to be initialized from this context
  6154. * because of lock dependencies (cgroup_lock -> cpu hotplug) but basically
  6155. * everything that doesn't depend on a specific mem_cgroup structure should
  6156. * be initialized from here.
  6157. */
  6158. static int __init mem_cgroup_init(void)
  6159. {
  6160. hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
  6161. enable_swap_cgroup();
  6162. mem_cgroup_soft_limit_tree_init();
  6163. memcg_stock_init();
  6164. return 0;
  6165. }
  6166. subsys_initcall(mem_cgroup_init);