memcontrol.c 111 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369
  1. /* memcontrol.c - Memory Controller
  2. *
  3. * Copyright IBM Corporation, 2007
  4. * Author Balbir Singh <balbir@linux.vnet.ibm.com>
  5. *
  6. * Copyright 2007 OpenVZ SWsoft Inc
  7. * Author: Pavel Emelianov <xemul@openvz.org>
  8. *
  9. * Memory thresholds
  10. * Copyright (C) 2009 Nokia Corporation
  11. * Author: Kirill A. Shutemov
  12. *
  13. * This program is free software; you can redistribute it and/or modify
  14. * it under the terms of the GNU General Public License as published by
  15. * the Free Software Foundation; either version 2 of the License, or
  16. * (at your option) any later version.
  17. *
  18. * This program is distributed in the hope that it will be useful,
  19. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  20. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  21. * GNU General Public License for more details.
  22. */
  23. #include <linux/res_counter.h>
  24. #include <linux/memcontrol.h>
  25. #include <linux/cgroup.h>
  26. #include <linux/mm.h>
  27. #include <linux/hugetlb.h>
  28. #include <linux/pagemap.h>
  29. #include <linux/smp.h>
  30. #include <linux/page-flags.h>
  31. #include <linux/backing-dev.h>
  32. #include <linux/bit_spinlock.h>
  33. #include <linux/rcupdate.h>
  34. #include <linux/limits.h>
  35. #include <linux/mutex.h>
  36. #include <linux/rbtree.h>
  37. #include <linux/slab.h>
  38. #include <linux/swap.h>
  39. #include <linux/swapops.h>
  40. #include <linux/spinlock.h>
  41. #include <linux/eventfd.h>
  42. #include <linux/sort.h>
  43. #include <linux/fs.h>
  44. #include <linux/seq_file.h>
  45. #include <linux/vmalloc.h>
  46. #include <linux/mm_inline.h>
  47. #include <linux/page_cgroup.h>
  48. #include <linux/cpu.h>
  49. #include "internal.h"
  50. #include <asm/uaccess.h>
  51. struct cgroup_subsys mem_cgroup_subsys __read_mostly;
  52. #define MEM_CGROUP_RECLAIM_RETRIES 5
  53. struct mem_cgroup *root_mem_cgroup __read_mostly;
  54. #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
  55. /* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */
  56. int do_swap_account __read_mostly;
  57. static int really_do_swap_account __initdata = 1; /* for remember boot option*/
  58. #else
  59. #define do_swap_account (0)
  60. #endif
  61. /*
  62. * Per memcg event counter is incremented at every pagein/pageout. This counter
  63. * is used for trigger some periodic events. This is straightforward and better
  64. * than using jiffies etc. to handle periodic memcg event.
  65. *
  66. * These values will be used as !((event) & ((1 <<(thresh)) - 1))
  67. */
  68. #define THRESHOLDS_EVENTS_THRESH (7) /* once in 128 */
  69. #define SOFTLIMIT_EVENTS_THRESH (10) /* once in 1024 */
  70. /*
  71. * Statistics for memory cgroup.
  72. */
  73. enum mem_cgroup_stat_index {
  74. /*
  75. * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
  76. */
  77. MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */
  78. MEM_CGROUP_STAT_RSS, /* # of pages charged as anon rss */
  79. MEM_CGROUP_STAT_FILE_MAPPED, /* # of pages charged as file rss */
  80. MEM_CGROUP_STAT_PGPGIN_COUNT, /* # of pages paged in */
  81. MEM_CGROUP_STAT_PGPGOUT_COUNT, /* # of pages paged out */
  82. MEM_CGROUP_STAT_SWAPOUT, /* # of pages, swapped out */
  83. MEM_CGROUP_EVENTS, /* incremented at every pagein/pageout */
  84. MEM_CGROUP_STAT_NSTATS,
  85. };
  86. struct mem_cgroup_stat_cpu {
  87. s64 count[MEM_CGROUP_STAT_NSTATS];
  88. };
  89. /*
  90. * per-zone information in memory controller.
  91. */
  92. struct mem_cgroup_per_zone {
  93. /*
  94. * spin_lock to protect the per cgroup LRU
  95. */
  96. struct list_head lists[NR_LRU_LISTS];
  97. unsigned long count[NR_LRU_LISTS];
  98. struct zone_reclaim_stat reclaim_stat;
  99. struct rb_node tree_node; /* RB tree node */
  100. unsigned long long usage_in_excess;/* Set to the value by which */
  101. /* the soft limit is exceeded*/
  102. bool on_tree;
  103. struct mem_cgroup *mem; /* Back pointer, we cannot */
  104. /* use container_of */
  105. };
  106. /* Macro for accessing counter */
  107. #define MEM_CGROUP_ZSTAT(mz, idx) ((mz)->count[(idx)])
  108. struct mem_cgroup_per_node {
  109. struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
  110. };
  111. struct mem_cgroup_lru_info {
  112. struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES];
  113. };
  114. /*
  115. * Cgroups above their limits are maintained in a RB-Tree, independent of
  116. * their hierarchy representation
  117. */
  118. struct mem_cgroup_tree_per_zone {
  119. struct rb_root rb_root;
  120. spinlock_t lock;
  121. };
  122. struct mem_cgroup_tree_per_node {
  123. struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
  124. };
  125. struct mem_cgroup_tree {
  126. struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
  127. };
  128. static struct mem_cgroup_tree soft_limit_tree __read_mostly;
  129. struct mem_cgroup_threshold {
  130. struct eventfd_ctx *eventfd;
  131. u64 threshold;
  132. };
  133. struct mem_cgroup_threshold_ary {
  134. /* An array index points to threshold just below usage. */
  135. atomic_t current_threshold;
  136. /* Size of entries[] */
  137. unsigned int size;
  138. /* Array of thresholds */
  139. struct mem_cgroup_threshold entries[0];
  140. };
  141. static void mem_cgroup_threshold(struct mem_cgroup *mem);
  142. /*
  143. * The memory controller data structure. The memory controller controls both
  144. * page cache and RSS per cgroup. We would eventually like to provide
  145. * statistics based on the statistics developed by Rik Van Riel for clock-pro,
  146. * to help the administrator determine what knobs to tune.
  147. *
  148. * TODO: Add a water mark for the memory controller. Reclaim will begin when
  149. * we hit the water mark. May be even add a low water mark, such that
  150. * no reclaim occurs from a cgroup at it's low water mark, this is
  151. * a feature that will be implemented much later in the future.
  152. */
  153. struct mem_cgroup {
  154. struct cgroup_subsys_state css;
  155. /*
  156. * the counter to account for memory usage
  157. */
  158. struct res_counter res;
  159. /*
  160. * the counter to account for mem+swap usage.
  161. */
  162. struct res_counter memsw;
  163. /*
  164. * Per cgroup active and inactive list, similar to the
  165. * per zone LRU lists.
  166. */
  167. struct mem_cgroup_lru_info info;
  168. /*
  169. protect against reclaim related member.
  170. */
  171. spinlock_t reclaim_param_lock;
  172. int prev_priority; /* for recording reclaim priority */
  173. /*
  174. * While reclaiming in a hierarchy, we cache the last child we
  175. * reclaimed from.
  176. */
  177. int last_scanned_child;
  178. /*
  179. * Should the accounting and control be hierarchical, per subtree?
  180. */
  181. bool use_hierarchy;
  182. atomic_t oom_lock;
  183. atomic_t refcnt;
  184. unsigned int swappiness;
  185. /* set when res.limit == memsw.limit */
  186. bool memsw_is_minimum;
  187. /* protect arrays of thresholds */
  188. struct mutex thresholds_lock;
  189. /* thresholds for memory usage. RCU-protected */
  190. struct mem_cgroup_threshold_ary *thresholds;
  191. /* thresholds for mem+swap usage. RCU-protected */
  192. struct mem_cgroup_threshold_ary *memsw_thresholds;
  193. /*
  194. * Should we move charges of a task when a task is moved into this
  195. * mem_cgroup ? And what type of charges should we move ?
  196. */
  197. unsigned long move_charge_at_immigrate;
  198. /*
  199. * percpu counter.
  200. */
  201. struct mem_cgroup_stat_cpu *stat;
  202. };
  203. /* Stuffs for move charges at task migration. */
  204. /*
  205. * Types of charges to be moved. "move_charge_at_immitgrate" is treated as a
  206. * left-shifted bitmap of these types.
  207. */
  208. enum move_type {
  209. MOVE_CHARGE_TYPE_ANON, /* private anonymous page and swap of it */
  210. NR_MOVE_TYPE,
  211. };
  212. /* "mc" and its members are protected by cgroup_mutex */
  213. static struct move_charge_struct {
  214. struct mem_cgroup *from;
  215. struct mem_cgroup *to;
  216. unsigned long precharge;
  217. unsigned long moved_charge;
  218. unsigned long moved_swap;
  219. struct task_struct *moving_task; /* a task moving charges */
  220. wait_queue_head_t waitq; /* a waitq for other context */
  221. } mc = {
  222. .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
  223. };
  224. /*
  225. * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
  226. * limit reclaim to prevent infinite loops, if they ever occur.
  227. */
  228. #define MEM_CGROUP_MAX_RECLAIM_LOOPS (100)
  229. #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS (2)
  230. enum charge_type {
  231. MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
  232. MEM_CGROUP_CHARGE_TYPE_MAPPED,
  233. MEM_CGROUP_CHARGE_TYPE_SHMEM, /* used by page migration of shmem */
  234. MEM_CGROUP_CHARGE_TYPE_FORCE, /* used by force_empty */
  235. MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
  236. MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */
  237. NR_CHARGE_TYPE,
  238. };
  239. /* only for here (for easy reading.) */
  240. #define PCGF_CACHE (1UL << PCG_CACHE)
  241. #define PCGF_USED (1UL << PCG_USED)
  242. #define PCGF_LOCK (1UL << PCG_LOCK)
  243. /* Not used, but added here for completeness */
  244. #define PCGF_ACCT (1UL << PCG_ACCT)
  245. /* for encoding cft->private value on file */
  246. #define _MEM (0)
  247. #define _MEMSWAP (1)
  248. #define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val))
  249. #define MEMFILE_TYPE(val) (((val) >> 16) & 0xffff)
  250. #define MEMFILE_ATTR(val) ((val) & 0xffff)
  251. /*
  252. * Reclaim flags for mem_cgroup_hierarchical_reclaim
  253. */
  254. #define MEM_CGROUP_RECLAIM_NOSWAP_BIT 0x0
  255. #define MEM_CGROUP_RECLAIM_NOSWAP (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
  256. #define MEM_CGROUP_RECLAIM_SHRINK_BIT 0x1
  257. #define MEM_CGROUP_RECLAIM_SHRINK (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
  258. #define MEM_CGROUP_RECLAIM_SOFT_BIT 0x2
  259. #define MEM_CGROUP_RECLAIM_SOFT (1 << MEM_CGROUP_RECLAIM_SOFT_BIT)
  260. static void mem_cgroup_get(struct mem_cgroup *mem);
  261. static void mem_cgroup_put(struct mem_cgroup *mem);
  262. static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem);
  263. static void drain_all_stock_async(void);
  264. static struct mem_cgroup_per_zone *
  265. mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid)
  266. {
  267. return &mem->info.nodeinfo[nid]->zoneinfo[zid];
  268. }
  269. struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *mem)
  270. {
  271. return &mem->css;
  272. }
  273. static struct mem_cgroup_per_zone *
  274. page_cgroup_zoneinfo(struct page_cgroup *pc)
  275. {
  276. struct mem_cgroup *mem = pc->mem_cgroup;
  277. int nid = page_cgroup_nid(pc);
  278. int zid = page_cgroup_zid(pc);
  279. if (!mem)
  280. return NULL;
  281. return mem_cgroup_zoneinfo(mem, nid, zid);
  282. }
  283. static struct mem_cgroup_tree_per_zone *
  284. soft_limit_tree_node_zone(int nid, int zid)
  285. {
  286. return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
  287. }
  288. static struct mem_cgroup_tree_per_zone *
  289. soft_limit_tree_from_page(struct page *page)
  290. {
  291. int nid = page_to_nid(page);
  292. int zid = page_zonenum(page);
  293. return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
  294. }
  295. static void
  296. __mem_cgroup_insert_exceeded(struct mem_cgroup *mem,
  297. struct mem_cgroup_per_zone *mz,
  298. struct mem_cgroup_tree_per_zone *mctz,
  299. unsigned long long new_usage_in_excess)
  300. {
  301. struct rb_node **p = &mctz->rb_root.rb_node;
  302. struct rb_node *parent = NULL;
  303. struct mem_cgroup_per_zone *mz_node;
  304. if (mz->on_tree)
  305. return;
  306. mz->usage_in_excess = new_usage_in_excess;
  307. if (!mz->usage_in_excess)
  308. return;
  309. while (*p) {
  310. parent = *p;
  311. mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
  312. tree_node);
  313. if (mz->usage_in_excess < mz_node->usage_in_excess)
  314. p = &(*p)->rb_left;
  315. /*
  316. * We can't avoid mem cgroups that are over their soft
  317. * limit by the same amount
  318. */
  319. else if (mz->usage_in_excess >= mz_node->usage_in_excess)
  320. p = &(*p)->rb_right;
  321. }
  322. rb_link_node(&mz->tree_node, parent, p);
  323. rb_insert_color(&mz->tree_node, &mctz->rb_root);
  324. mz->on_tree = true;
  325. }
  326. static void
  327. __mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
  328. struct mem_cgroup_per_zone *mz,
  329. struct mem_cgroup_tree_per_zone *mctz)
  330. {
  331. if (!mz->on_tree)
  332. return;
  333. rb_erase(&mz->tree_node, &mctz->rb_root);
  334. mz->on_tree = false;
  335. }
  336. static void
  337. mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
  338. struct mem_cgroup_per_zone *mz,
  339. struct mem_cgroup_tree_per_zone *mctz)
  340. {
  341. spin_lock(&mctz->lock);
  342. __mem_cgroup_remove_exceeded(mem, mz, mctz);
  343. spin_unlock(&mctz->lock);
  344. }
  345. static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page)
  346. {
  347. unsigned long long excess;
  348. struct mem_cgroup_per_zone *mz;
  349. struct mem_cgroup_tree_per_zone *mctz;
  350. int nid = page_to_nid(page);
  351. int zid = page_zonenum(page);
  352. mctz = soft_limit_tree_from_page(page);
  353. /*
  354. * Necessary to update all ancestors when hierarchy is used.
  355. * because their event counter is not touched.
  356. */
  357. for (; mem; mem = parent_mem_cgroup(mem)) {
  358. mz = mem_cgroup_zoneinfo(mem, nid, zid);
  359. excess = res_counter_soft_limit_excess(&mem->res);
  360. /*
  361. * We have to update the tree if mz is on RB-tree or
  362. * mem is over its softlimit.
  363. */
  364. if (excess || mz->on_tree) {
  365. spin_lock(&mctz->lock);
  366. /* if on-tree, remove it */
  367. if (mz->on_tree)
  368. __mem_cgroup_remove_exceeded(mem, mz, mctz);
  369. /*
  370. * Insert again. mz->usage_in_excess will be updated.
  371. * If excess is 0, no tree ops.
  372. */
  373. __mem_cgroup_insert_exceeded(mem, mz, mctz, excess);
  374. spin_unlock(&mctz->lock);
  375. }
  376. }
  377. }
  378. static void mem_cgroup_remove_from_trees(struct mem_cgroup *mem)
  379. {
  380. int node, zone;
  381. struct mem_cgroup_per_zone *mz;
  382. struct mem_cgroup_tree_per_zone *mctz;
  383. for_each_node_state(node, N_POSSIBLE) {
  384. for (zone = 0; zone < MAX_NR_ZONES; zone++) {
  385. mz = mem_cgroup_zoneinfo(mem, node, zone);
  386. mctz = soft_limit_tree_node_zone(node, zone);
  387. mem_cgroup_remove_exceeded(mem, mz, mctz);
  388. }
  389. }
  390. }
  391. static inline unsigned long mem_cgroup_get_excess(struct mem_cgroup *mem)
  392. {
  393. return res_counter_soft_limit_excess(&mem->res) >> PAGE_SHIFT;
  394. }
  395. static struct mem_cgroup_per_zone *
  396. __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
  397. {
  398. struct rb_node *rightmost = NULL;
  399. struct mem_cgroup_per_zone *mz;
  400. retry:
  401. mz = NULL;
  402. rightmost = rb_last(&mctz->rb_root);
  403. if (!rightmost)
  404. goto done; /* Nothing to reclaim from */
  405. mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
  406. /*
  407. * Remove the node now but someone else can add it back,
  408. * we will to add it back at the end of reclaim to its correct
  409. * position in the tree.
  410. */
  411. __mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
  412. if (!res_counter_soft_limit_excess(&mz->mem->res) ||
  413. !css_tryget(&mz->mem->css))
  414. goto retry;
  415. done:
  416. return mz;
  417. }
  418. static struct mem_cgroup_per_zone *
  419. mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
  420. {
  421. struct mem_cgroup_per_zone *mz;
  422. spin_lock(&mctz->lock);
  423. mz = __mem_cgroup_largest_soft_limit_node(mctz);
  424. spin_unlock(&mctz->lock);
  425. return mz;
  426. }
  427. static s64 mem_cgroup_read_stat(struct mem_cgroup *mem,
  428. enum mem_cgroup_stat_index idx)
  429. {
  430. int cpu;
  431. s64 val = 0;
  432. for_each_possible_cpu(cpu)
  433. val += per_cpu(mem->stat->count[idx], cpu);
  434. return val;
  435. }
  436. static s64 mem_cgroup_local_usage(struct mem_cgroup *mem)
  437. {
  438. s64 ret;
  439. ret = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS);
  440. ret += mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE);
  441. return ret;
  442. }
  443. static void mem_cgroup_swap_statistics(struct mem_cgroup *mem,
  444. bool charge)
  445. {
  446. int val = (charge) ? 1 : -1;
  447. this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_SWAPOUT], val);
  448. }
  449. static void mem_cgroup_charge_statistics(struct mem_cgroup *mem,
  450. struct page_cgroup *pc,
  451. bool charge)
  452. {
  453. int val = (charge) ? 1 : -1;
  454. preempt_disable();
  455. if (PageCgroupCache(pc))
  456. __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_CACHE], val);
  457. else
  458. __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_RSS], val);
  459. if (charge)
  460. __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGIN_COUNT]);
  461. else
  462. __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGOUT_COUNT]);
  463. __this_cpu_inc(mem->stat->count[MEM_CGROUP_EVENTS]);
  464. preempt_enable();
  465. }
  466. static unsigned long mem_cgroup_get_local_zonestat(struct mem_cgroup *mem,
  467. enum lru_list idx)
  468. {
  469. int nid, zid;
  470. struct mem_cgroup_per_zone *mz;
  471. u64 total = 0;
  472. for_each_online_node(nid)
  473. for (zid = 0; zid < MAX_NR_ZONES; zid++) {
  474. mz = mem_cgroup_zoneinfo(mem, nid, zid);
  475. total += MEM_CGROUP_ZSTAT(mz, idx);
  476. }
  477. return total;
  478. }
  479. static bool __memcg_event_check(struct mem_cgroup *mem, int event_mask_shift)
  480. {
  481. s64 val;
  482. val = this_cpu_read(mem->stat->count[MEM_CGROUP_EVENTS]);
  483. return !(val & ((1 << event_mask_shift) - 1));
  484. }
  485. /*
  486. * Check events in order.
  487. *
  488. */
  489. static void memcg_check_events(struct mem_cgroup *mem, struct page *page)
  490. {
  491. /* threshold event is triggered in finer grain than soft limit */
  492. if (unlikely(__memcg_event_check(mem, THRESHOLDS_EVENTS_THRESH))) {
  493. mem_cgroup_threshold(mem);
  494. if (unlikely(__memcg_event_check(mem, SOFTLIMIT_EVENTS_THRESH)))
  495. mem_cgroup_update_tree(mem, page);
  496. }
  497. }
  498. static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
  499. {
  500. return container_of(cgroup_subsys_state(cont,
  501. mem_cgroup_subsys_id), struct mem_cgroup,
  502. css);
  503. }
  504. struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
  505. {
  506. /*
  507. * mm_update_next_owner() may clear mm->owner to NULL
  508. * if it races with swapoff, page migration, etc.
  509. * So this can be called with p == NULL.
  510. */
  511. if (unlikely(!p))
  512. return NULL;
  513. return container_of(task_subsys_state(p, mem_cgroup_subsys_id),
  514. struct mem_cgroup, css);
  515. }
  516. static struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
  517. {
  518. struct mem_cgroup *mem = NULL;
  519. if (!mm)
  520. return NULL;
  521. /*
  522. * Because we have no locks, mm->owner's may be being moved to other
  523. * cgroup. We use css_tryget() here even if this looks
  524. * pessimistic (rather than adding locks here).
  525. */
  526. rcu_read_lock();
  527. do {
  528. mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
  529. if (unlikely(!mem))
  530. break;
  531. } while (!css_tryget(&mem->css));
  532. rcu_read_unlock();
  533. return mem;
  534. }
  535. /*
  536. * Call callback function against all cgroup under hierarchy tree.
  537. */
  538. static int mem_cgroup_walk_tree(struct mem_cgroup *root, void *data,
  539. int (*func)(struct mem_cgroup *, void *))
  540. {
  541. int found, ret, nextid;
  542. struct cgroup_subsys_state *css;
  543. struct mem_cgroup *mem;
  544. if (!root->use_hierarchy)
  545. return (*func)(root, data);
  546. nextid = 1;
  547. do {
  548. ret = 0;
  549. mem = NULL;
  550. rcu_read_lock();
  551. css = css_get_next(&mem_cgroup_subsys, nextid, &root->css,
  552. &found);
  553. if (css && css_tryget(css))
  554. mem = container_of(css, struct mem_cgroup, css);
  555. rcu_read_unlock();
  556. if (mem) {
  557. ret = (*func)(mem, data);
  558. css_put(&mem->css);
  559. }
  560. nextid = found + 1;
  561. } while (!ret && css);
  562. return ret;
  563. }
  564. static inline bool mem_cgroup_is_root(struct mem_cgroup *mem)
  565. {
  566. return (mem == root_mem_cgroup);
  567. }
  568. /*
  569. * Following LRU functions are allowed to be used without PCG_LOCK.
  570. * Operations are called by routine of global LRU independently from memcg.
  571. * What we have to take care of here is validness of pc->mem_cgroup.
  572. *
  573. * Changes to pc->mem_cgroup happens when
  574. * 1. charge
  575. * 2. moving account
  576. * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
  577. * It is added to LRU before charge.
  578. * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
  579. * When moving account, the page is not on LRU. It's isolated.
  580. */
  581. void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru)
  582. {
  583. struct page_cgroup *pc;
  584. struct mem_cgroup_per_zone *mz;
  585. if (mem_cgroup_disabled())
  586. return;
  587. pc = lookup_page_cgroup(page);
  588. /* can happen while we handle swapcache. */
  589. if (!TestClearPageCgroupAcctLRU(pc))
  590. return;
  591. VM_BUG_ON(!pc->mem_cgroup);
  592. /*
  593. * We don't check PCG_USED bit. It's cleared when the "page" is finally
  594. * removed from global LRU.
  595. */
  596. mz = page_cgroup_zoneinfo(pc);
  597. MEM_CGROUP_ZSTAT(mz, lru) -= 1;
  598. if (mem_cgroup_is_root(pc->mem_cgroup))
  599. return;
  600. VM_BUG_ON(list_empty(&pc->lru));
  601. list_del_init(&pc->lru);
  602. return;
  603. }
  604. void mem_cgroup_del_lru(struct page *page)
  605. {
  606. mem_cgroup_del_lru_list(page, page_lru(page));
  607. }
  608. void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru)
  609. {
  610. struct mem_cgroup_per_zone *mz;
  611. struct page_cgroup *pc;
  612. if (mem_cgroup_disabled())
  613. return;
  614. pc = lookup_page_cgroup(page);
  615. /*
  616. * Used bit is set without atomic ops but after smp_wmb().
  617. * For making pc->mem_cgroup visible, insert smp_rmb() here.
  618. */
  619. smp_rmb();
  620. /* unused or root page is not rotated. */
  621. if (!PageCgroupUsed(pc) || mem_cgroup_is_root(pc->mem_cgroup))
  622. return;
  623. mz = page_cgroup_zoneinfo(pc);
  624. list_move(&pc->lru, &mz->lists[lru]);
  625. }
  626. void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru)
  627. {
  628. struct page_cgroup *pc;
  629. struct mem_cgroup_per_zone *mz;
  630. if (mem_cgroup_disabled())
  631. return;
  632. pc = lookup_page_cgroup(page);
  633. VM_BUG_ON(PageCgroupAcctLRU(pc));
  634. /*
  635. * Used bit is set without atomic ops but after smp_wmb().
  636. * For making pc->mem_cgroup visible, insert smp_rmb() here.
  637. */
  638. smp_rmb();
  639. if (!PageCgroupUsed(pc))
  640. return;
  641. mz = page_cgroup_zoneinfo(pc);
  642. MEM_CGROUP_ZSTAT(mz, lru) += 1;
  643. SetPageCgroupAcctLRU(pc);
  644. if (mem_cgroup_is_root(pc->mem_cgroup))
  645. return;
  646. list_add(&pc->lru, &mz->lists[lru]);
  647. }
  648. /*
  649. * At handling SwapCache, pc->mem_cgroup may be changed while it's linked to
  650. * lru because the page may.be reused after it's fully uncharged (because of
  651. * SwapCache behavior).To handle that, unlink page_cgroup from LRU when charge
  652. * it again. This function is only used to charge SwapCache. It's done under
  653. * lock_page and expected that zone->lru_lock is never held.
  654. */
  655. static void mem_cgroup_lru_del_before_commit_swapcache(struct page *page)
  656. {
  657. unsigned long flags;
  658. struct zone *zone = page_zone(page);
  659. struct page_cgroup *pc = lookup_page_cgroup(page);
  660. spin_lock_irqsave(&zone->lru_lock, flags);
  661. /*
  662. * Forget old LRU when this page_cgroup is *not* used. This Used bit
  663. * is guarded by lock_page() because the page is SwapCache.
  664. */
  665. if (!PageCgroupUsed(pc))
  666. mem_cgroup_del_lru_list(page, page_lru(page));
  667. spin_unlock_irqrestore(&zone->lru_lock, flags);
  668. }
  669. static void mem_cgroup_lru_add_after_commit_swapcache(struct page *page)
  670. {
  671. unsigned long flags;
  672. struct zone *zone = page_zone(page);
  673. struct page_cgroup *pc = lookup_page_cgroup(page);
  674. spin_lock_irqsave(&zone->lru_lock, flags);
  675. /* link when the page is linked to LRU but page_cgroup isn't */
  676. if (PageLRU(page) && !PageCgroupAcctLRU(pc))
  677. mem_cgroup_add_lru_list(page, page_lru(page));
  678. spin_unlock_irqrestore(&zone->lru_lock, flags);
  679. }
  680. void mem_cgroup_move_lists(struct page *page,
  681. enum lru_list from, enum lru_list to)
  682. {
  683. if (mem_cgroup_disabled())
  684. return;
  685. mem_cgroup_del_lru_list(page, from);
  686. mem_cgroup_add_lru_list(page, to);
  687. }
  688. int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
  689. {
  690. int ret;
  691. struct mem_cgroup *curr = NULL;
  692. task_lock(task);
  693. rcu_read_lock();
  694. curr = try_get_mem_cgroup_from_mm(task->mm);
  695. rcu_read_unlock();
  696. task_unlock(task);
  697. if (!curr)
  698. return 0;
  699. /*
  700. * We should check use_hierarchy of "mem" not "curr". Because checking
  701. * use_hierarchy of "curr" here make this function true if hierarchy is
  702. * enabled in "curr" and "curr" is a child of "mem" in *cgroup*
  703. * hierarchy(even if use_hierarchy is disabled in "mem").
  704. */
  705. rcu_read_lock();
  706. if (mem->use_hierarchy)
  707. ret = css_is_ancestor(&curr->css, &mem->css);
  708. else
  709. ret = (curr == mem);
  710. rcu_read_unlock();
  711. css_put(&curr->css);
  712. return ret;
  713. }
  714. /*
  715. * prev_priority control...this will be used in memory reclaim path.
  716. */
  717. int mem_cgroup_get_reclaim_priority(struct mem_cgroup *mem)
  718. {
  719. int prev_priority;
  720. spin_lock(&mem->reclaim_param_lock);
  721. prev_priority = mem->prev_priority;
  722. spin_unlock(&mem->reclaim_param_lock);
  723. return prev_priority;
  724. }
  725. void mem_cgroup_note_reclaim_priority(struct mem_cgroup *mem, int priority)
  726. {
  727. spin_lock(&mem->reclaim_param_lock);
  728. if (priority < mem->prev_priority)
  729. mem->prev_priority = priority;
  730. spin_unlock(&mem->reclaim_param_lock);
  731. }
  732. void mem_cgroup_record_reclaim_priority(struct mem_cgroup *mem, int priority)
  733. {
  734. spin_lock(&mem->reclaim_param_lock);
  735. mem->prev_priority = priority;
  736. spin_unlock(&mem->reclaim_param_lock);
  737. }
  738. static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_pages)
  739. {
  740. unsigned long active;
  741. unsigned long inactive;
  742. unsigned long gb;
  743. unsigned long inactive_ratio;
  744. inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_ANON);
  745. active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_ANON);
  746. gb = (inactive + active) >> (30 - PAGE_SHIFT);
  747. if (gb)
  748. inactive_ratio = int_sqrt(10 * gb);
  749. else
  750. inactive_ratio = 1;
  751. if (present_pages) {
  752. present_pages[0] = inactive;
  753. present_pages[1] = active;
  754. }
  755. return inactive_ratio;
  756. }
  757. int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg)
  758. {
  759. unsigned long active;
  760. unsigned long inactive;
  761. unsigned long present_pages[2];
  762. unsigned long inactive_ratio;
  763. inactive_ratio = calc_inactive_ratio(memcg, present_pages);
  764. inactive = present_pages[0];
  765. active = present_pages[1];
  766. if (inactive * inactive_ratio < active)
  767. return 1;
  768. return 0;
  769. }
  770. int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg)
  771. {
  772. unsigned long active;
  773. unsigned long inactive;
  774. inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_FILE);
  775. active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_FILE);
  776. return (active > inactive);
  777. }
  778. unsigned long mem_cgroup_zone_nr_pages(struct mem_cgroup *memcg,
  779. struct zone *zone,
  780. enum lru_list lru)
  781. {
  782. int nid = zone->zone_pgdat->node_id;
  783. int zid = zone_idx(zone);
  784. struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
  785. return MEM_CGROUP_ZSTAT(mz, lru);
  786. }
  787. struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg,
  788. struct zone *zone)
  789. {
  790. int nid = zone->zone_pgdat->node_id;
  791. int zid = zone_idx(zone);
  792. struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
  793. return &mz->reclaim_stat;
  794. }
  795. struct zone_reclaim_stat *
  796. mem_cgroup_get_reclaim_stat_from_page(struct page *page)
  797. {
  798. struct page_cgroup *pc;
  799. struct mem_cgroup_per_zone *mz;
  800. if (mem_cgroup_disabled())
  801. return NULL;
  802. pc = lookup_page_cgroup(page);
  803. /*
  804. * Used bit is set without atomic ops but after smp_wmb().
  805. * For making pc->mem_cgroup visible, insert smp_rmb() here.
  806. */
  807. smp_rmb();
  808. if (!PageCgroupUsed(pc))
  809. return NULL;
  810. mz = page_cgroup_zoneinfo(pc);
  811. if (!mz)
  812. return NULL;
  813. return &mz->reclaim_stat;
  814. }
  815. unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
  816. struct list_head *dst,
  817. unsigned long *scanned, int order,
  818. int mode, struct zone *z,
  819. struct mem_cgroup *mem_cont,
  820. int active, int file)
  821. {
  822. unsigned long nr_taken = 0;
  823. struct page *page;
  824. unsigned long scan;
  825. LIST_HEAD(pc_list);
  826. struct list_head *src;
  827. struct page_cgroup *pc, *tmp;
  828. int nid = z->zone_pgdat->node_id;
  829. int zid = zone_idx(z);
  830. struct mem_cgroup_per_zone *mz;
  831. int lru = LRU_FILE * file + active;
  832. int ret;
  833. BUG_ON(!mem_cont);
  834. mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
  835. src = &mz->lists[lru];
  836. scan = 0;
  837. list_for_each_entry_safe_reverse(pc, tmp, src, lru) {
  838. if (scan >= nr_to_scan)
  839. break;
  840. page = pc->page;
  841. if (unlikely(!PageCgroupUsed(pc)))
  842. continue;
  843. if (unlikely(!PageLRU(page)))
  844. continue;
  845. scan++;
  846. ret = __isolate_lru_page(page, mode, file);
  847. switch (ret) {
  848. case 0:
  849. list_move(&page->lru, dst);
  850. mem_cgroup_del_lru(page);
  851. nr_taken++;
  852. break;
  853. case -EBUSY:
  854. /* we don't affect global LRU but rotate in our LRU */
  855. mem_cgroup_rotate_lru_list(page, page_lru(page));
  856. break;
  857. default:
  858. break;
  859. }
  860. }
  861. *scanned = scan;
  862. return nr_taken;
  863. }
  864. #define mem_cgroup_from_res_counter(counter, member) \
  865. container_of(counter, struct mem_cgroup, member)
  866. static bool mem_cgroup_check_under_limit(struct mem_cgroup *mem)
  867. {
  868. if (do_swap_account) {
  869. if (res_counter_check_under_limit(&mem->res) &&
  870. res_counter_check_under_limit(&mem->memsw))
  871. return true;
  872. } else
  873. if (res_counter_check_under_limit(&mem->res))
  874. return true;
  875. return false;
  876. }
  877. static unsigned int get_swappiness(struct mem_cgroup *memcg)
  878. {
  879. struct cgroup *cgrp = memcg->css.cgroup;
  880. unsigned int swappiness;
  881. /* root ? */
  882. if (cgrp->parent == NULL)
  883. return vm_swappiness;
  884. spin_lock(&memcg->reclaim_param_lock);
  885. swappiness = memcg->swappiness;
  886. spin_unlock(&memcg->reclaim_param_lock);
  887. return swappiness;
  888. }
  889. static int mem_cgroup_count_children_cb(struct mem_cgroup *mem, void *data)
  890. {
  891. int *val = data;
  892. (*val)++;
  893. return 0;
  894. }
  895. /**
  896. * mem_cgroup_print_oom_info: Called from OOM with tasklist_lock held in read mode.
  897. * @memcg: The memory cgroup that went over limit
  898. * @p: Task that is going to be killed
  899. *
  900. * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
  901. * enabled
  902. */
  903. void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
  904. {
  905. struct cgroup *task_cgrp;
  906. struct cgroup *mem_cgrp;
  907. /*
  908. * Need a buffer in BSS, can't rely on allocations. The code relies
  909. * on the assumption that OOM is serialized for memory controller.
  910. * If this assumption is broken, revisit this code.
  911. */
  912. static char memcg_name[PATH_MAX];
  913. int ret;
  914. if (!memcg || !p)
  915. return;
  916. rcu_read_lock();
  917. mem_cgrp = memcg->css.cgroup;
  918. task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);
  919. ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
  920. if (ret < 0) {
  921. /*
  922. * Unfortunately, we are unable to convert to a useful name
  923. * But we'll still print out the usage information
  924. */
  925. rcu_read_unlock();
  926. goto done;
  927. }
  928. rcu_read_unlock();
  929. printk(KERN_INFO "Task in %s killed", memcg_name);
  930. rcu_read_lock();
  931. ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
  932. if (ret < 0) {
  933. rcu_read_unlock();
  934. goto done;
  935. }
  936. rcu_read_unlock();
  937. /*
  938. * Continues from above, so we don't need an KERN_ level
  939. */
  940. printk(KERN_CONT " as a result of limit of %s\n", memcg_name);
  941. done:
  942. printk(KERN_INFO "memory: usage %llukB, limit %llukB, failcnt %llu\n",
  943. res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
  944. res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
  945. res_counter_read_u64(&memcg->res, RES_FAILCNT));
  946. printk(KERN_INFO "memory+swap: usage %llukB, limit %llukB, "
  947. "failcnt %llu\n",
  948. res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
  949. res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
  950. res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
  951. }
  952. /*
  953. * This function returns the number of memcg under hierarchy tree. Returns
  954. * 1(self count) if no children.
  955. */
  956. static int mem_cgroup_count_children(struct mem_cgroup *mem)
  957. {
  958. int num = 0;
  959. mem_cgroup_walk_tree(mem, &num, mem_cgroup_count_children_cb);
  960. return num;
  961. }
  962. /*
  963. * Visit the first child (need not be the first child as per the ordering
  964. * of the cgroup list, since we track last_scanned_child) of @mem and use
  965. * that to reclaim free pages from.
  966. */
  967. static struct mem_cgroup *
  968. mem_cgroup_select_victim(struct mem_cgroup *root_mem)
  969. {
  970. struct mem_cgroup *ret = NULL;
  971. struct cgroup_subsys_state *css;
  972. int nextid, found;
  973. if (!root_mem->use_hierarchy) {
  974. css_get(&root_mem->css);
  975. ret = root_mem;
  976. }
  977. while (!ret) {
  978. rcu_read_lock();
  979. nextid = root_mem->last_scanned_child + 1;
  980. css = css_get_next(&mem_cgroup_subsys, nextid, &root_mem->css,
  981. &found);
  982. if (css && css_tryget(css))
  983. ret = container_of(css, struct mem_cgroup, css);
  984. rcu_read_unlock();
  985. /* Updates scanning parameter */
  986. spin_lock(&root_mem->reclaim_param_lock);
  987. if (!css) {
  988. /* this means start scan from ID:1 */
  989. root_mem->last_scanned_child = 0;
  990. } else
  991. root_mem->last_scanned_child = found;
  992. spin_unlock(&root_mem->reclaim_param_lock);
  993. }
  994. return ret;
  995. }
  996. /*
  997. * Scan the hierarchy if needed to reclaim memory. We remember the last child
  998. * we reclaimed from, so that we don't end up penalizing one child extensively
  999. * based on its position in the children list.
  1000. *
  1001. * root_mem is the original ancestor that we've been reclaim from.
  1002. *
  1003. * We give up and return to the caller when we visit root_mem twice.
  1004. * (other groups can be removed while we're walking....)
  1005. *
  1006. * If shrink==true, for avoiding to free too much, this returns immedieately.
  1007. */
  1008. static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
  1009. struct zone *zone,
  1010. gfp_t gfp_mask,
  1011. unsigned long reclaim_options)
  1012. {
  1013. struct mem_cgroup *victim;
  1014. int ret, total = 0;
  1015. int loop = 0;
  1016. bool noswap = reclaim_options & MEM_CGROUP_RECLAIM_NOSWAP;
  1017. bool shrink = reclaim_options & MEM_CGROUP_RECLAIM_SHRINK;
  1018. bool check_soft = reclaim_options & MEM_CGROUP_RECLAIM_SOFT;
  1019. unsigned long excess = mem_cgroup_get_excess(root_mem);
  1020. /* If memsw_is_minimum==1, swap-out is of-no-use. */
  1021. if (root_mem->memsw_is_minimum)
  1022. noswap = true;
  1023. while (1) {
  1024. victim = mem_cgroup_select_victim(root_mem);
  1025. if (victim == root_mem) {
  1026. loop++;
  1027. if (loop >= 1)
  1028. drain_all_stock_async();
  1029. if (loop >= 2) {
  1030. /*
  1031. * If we have not been able to reclaim
  1032. * anything, it might because there are
  1033. * no reclaimable pages under this hierarchy
  1034. */
  1035. if (!check_soft || !total) {
  1036. css_put(&victim->css);
  1037. break;
  1038. }
  1039. /*
  1040. * We want to do more targetted reclaim.
  1041. * excess >> 2 is not to excessive so as to
  1042. * reclaim too much, nor too less that we keep
  1043. * coming back to reclaim from this cgroup
  1044. */
  1045. if (total >= (excess >> 2) ||
  1046. (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) {
  1047. css_put(&victim->css);
  1048. break;
  1049. }
  1050. }
  1051. }
  1052. if (!mem_cgroup_local_usage(victim)) {
  1053. /* this cgroup's local usage == 0 */
  1054. css_put(&victim->css);
  1055. continue;
  1056. }
  1057. /* we use swappiness of local cgroup */
  1058. if (check_soft)
  1059. ret = mem_cgroup_shrink_node_zone(victim, gfp_mask,
  1060. noswap, get_swappiness(victim), zone,
  1061. zone->zone_pgdat->node_id);
  1062. else
  1063. ret = try_to_free_mem_cgroup_pages(victim, gfp_mask,
  1064. noswap, get_swappiness(victim));
  1065. css_put(&victim->css);
  1066. /*
  1067. * At shrinking usage, we can't check we should stop here or
  1068. * reclaim more. It's depends on callers. last_scanned_child
  1069. * will work enough for keeping fairness under tree.
  1070. */
  1071. if (shrink)
  1072. return ret;
  1073. total += ret;
  1074. if (check_soft) {
  1075. if (res_counter_check_under_soft_limit(&root_mem->res))
  1076. return total;
  1077. } else if (mem_cgroup_check_under_limit(root_mem))
  1078. return 1 + total;
  1079. }
  1080. return total;
  1081. }
  1082. static int mem_cgroup_oom_lock_cb(struct mem_cgroup *mem, void *data)
  1083. {
  1084. int *val = (int *)data;
  1085. int x;
  1086. /*
  1087. * Logically, we can stop scanning immediately when we find
  1088. * a memcg is already locked. But condidering unlock ops and
  1089. * creation/removal of memcg, scan-all is simple operation.
  1090. */
  1091. x = atomic_inc_return(&mem->oom_lock);
  1092. *val = max(x, *val);
  1093. return 0;
  1094. }
  1095. /*
  1096. * Check OOM-Killer is already running under our hierarchy.
  1097. * If someone is running, return false.
  1098. */
  1099. static bool mem_cgroup_oom_lock(struct mem_cgroup *mem)
  1100. {
  1101. int lock_count = 0;
  1102. mem_cgroup_walk_tree(mem, &lock_count, mem_cgroup_oom_lock_cb);
  1103. if (lock_count == 1)
  1104. return true;
  1105. return false;
  1106. }
  1107. static int mem_cgroup_oom_unlock_cb(struct mem_cgroup *mem, void *data)
  1108. {
  1109. /*
  1110. * When a new child is created while the hierarchy is under oom,
  1111. * mem_cgroup_oom_lock() may not be called. We have to use
  1112. * atomic_add_unless() here.
  1113. */
  1114. atomic_add_unless(&mem->oom_lock, -1, 0);
  1115. return 0;
  1116. }
  1117. static void mem_cgroup_oom_unlock(struct mem_cgroup *mem)
  1118. {
  1119. mem_cgroup_walk_tree(mem, NULL, mem_cgroup_oom_unlock_cb);
  1120. }
  1121. static DEFINE_MUTEX(memcg_oom_mutex);
  1122. static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
  1123. /*
  1124. * try to call OOM killer. returns false if we should exit memory-reclaim loop.
  1125. */
  1126. bool mem_cgroup_handle_oom(struct mem_cgroup *mem, gfp_t mask)
  1127. {
  1128. DEFINE_WAIT(wait);
  1129. bool locked;
  1130. /* At first, try to OOM lock hierarchy under mem.*/
  1131. mutex_lock(&memcg_oom_mutex);
  1132. locked = mem_cgroup_oom_lock(mem);
  1133. /*
  1134. * Even if signal_pending(), we can't quit charge() loop without
  1135. * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL
  1136. * under OOM is always welcomed, use TASK_KILLABLE here.
  1137. */
  1138. if (!locked)
  1139. prepare_to_wait(&memcg_oom_waitq, &wait, TASK_KILLABLE);
  1140. mutex_unlock(&memcg_oom_mutex);
  1141. if (locked)
  1142. mem_cgroup_out_of_memory(mem, mask);
  1143. else {
  1144. schedule();
  1145. finish_wait(&memcg_oom_waitq, &wait);
  1146. }
  1147. mutex_lock(&memcg_oom_mutex);
  1148. mem_cgroup_oom_unlock(mem);
  1149. /*
  1150. * Here, we use global waitq .....more fine grained waitq ?
  1151. * Assume following hierarchy.
  1152. * A/
  1153. * 01
  1154. * 02
  1155. * assume OOM happens both in A and 01 at the same time. Tthey are
  1156. * mutually exclusive by lock. (kill in 01 helps A.)
  1157. * When we use per memcg waitq, we have to wake up waiters on A and 02
  1158. * in addtion to waiters on 01. We use global waitq for avoiding mess.
  1159. * It will not be a big problem.
  1160. * (And a task may be moved to other groups while it's waiting for OOM.)
  1161. */
  1162. wake_up_all(&memcg_oom_waitq);
  1163. mutex_unlock(&memcg_oom_mutex);
  1164. if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current))
  1165. return false;
  1166. /* Give chance to dying process */
  1167. schedule_timeout(1);
  1168. return true;
  1169. }
  1170. /*
  1171. * Currently used to update mapped file statistics, but the routine can be
  1172. * generalized to update other statistics as well.
  1173. */
  1174. void mem_cgroup_update_file_mapped(struct page *page, int val)
  1175. {
  1176. struct mem_cgroup *mem;
  1177. struct page_cgroup *pc;
  1178. pc = lookup_page_cgroup(page);
  1179. if (unlikely(!pc))
  1180. return;
  1181. lock_page_cgroup(pc);
  1182. mem = pc->mem_cgroup;
  1183. if (!mem || !PageCgroupUsed(pc))
  1184. goto done;
  1185. /*
  1186. * Preemption is already disabled. We can use __this_cpu_xxx
  1187. */
  1188. if (val > 0) {
  1189. __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
  1190. SetPageCgroupFileMapped(pc);
  1191. } else {
  1192. __this_cpu_dec(mem->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
  1193. ClearPageCgroupFileMapped(pc);
  1194. }
  1195. done:
  1196. unlock_page_cgroup(pc);
  1197. }
  1198. /*
  1199. * size of first charge trial. "32" comes from vmscan.c's magic value.
  1200. * TODO: maybe necessary to use big numbers in big irons.
  1201. */
  1202. #define CHARGE_SIZE (32 * PAGE_SIZE)
  1203. struct memcg_stock_pcp {
  1204. struct mem_cgroup *cached; /* this never be root cgroup */
  1205. int charge;
  1206. struct work_struct work;
  1207. };
  1208. static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
  1209. static atomic_t memcg_drain_count;
  1210. /*
  1211. * Try to consume stocked charge on this cpu. If success, PAGE_SIZE is consumed
  1212. * from local stock and true is returned. If the stock is 0 or charges from a
  1213. * cgroup which is not current target, returns false. This stock will be
  1214. * refilled.
  1215. */
  1216. static bool consume_stock(struct mem_cgroup *mem)
  1217. {
  1218. struct memcg_stock_pcp *stock;
  1219. bool ret = true;
  1220. stock = &get_cpu_var(memcg_stock);
  1221. if (mem == stock->cached && stock->charge)
  1222. stock->charge -= PAGE_SIZE;
  1223. else /* need to call res_counter_charge */
  1224. ret = false;
  1225. put_cpu_var(memcg_stock);
  1226. return ret;
  1227. }
  1228. /*
  1229. * Returns stocks cached in percpu to res_counter and reset cached information.
  1230. */
  1231. static void drain_stock(struct memcg_stock_pcp *stock)
  1232. {
  1233. struct mem_cgroup *old = stock->cached;
  1234. if (stock->charge) {
  1235. res_counter_uncharge(&old->res, stock->charge);
  1236. if (do_swap_account)
  1237. res_counter_uncharge(&old->memsw, stock->charge);
  1238. }
  1239. stock->cached = NULL;
  1240. stock->charge = 0;
  1241. }
  1242. /*
  1243. * This must be called under preempt disabled or must be called by
  1244. * a thread which is pinned to local cpu.
  1245. */
  1246. static void drain_local_stock(struct work_struct *dummy)
  1247. {
  1248. struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
  1249. drain_stock(stock);
  1250. }
  1251. /*
  1252. * Cache charges(val) which is from res_counter, to local per_cpu area.
  1253. * This will be consumed by consumt_stock() function, later.
  1254. */
  1255. static void refill_stock(struct mem_cgroup *mem, int val)
  1256. {
  1257. struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
  1258. if (stock->cached != mem) { /* reset if necessary */
  1259. drain_stock(stock);
  1260. stock->cached = mem;
  1261. }
  1262. stock->charge += val;
  1263. put_cpu_var(memcg_stock);
  1264. }
  1265. /*
  1266. * Tries to drain stocked charges in other cpus. This function is asynchronous
  1267. * and just put a work per cpu for draining localy on each cpu. Caller can
  1268. * expects some charges will be back to res_counter later but cannot wait for
  1269. * it.
  1270. */
  1271. static void drain_all_stock_async(void)
  1272. {
  1273. int cpu;
  1274. /* This function is for scheduling "drain" in asynchronous way.
  1275. * The result of "drain" is not directly handled by callers. Then,
  1276. * if someone is calling drain, we don't have to call drain more.
  1277. * Anyway, WORK_STRUCT_PENDING check in queue_work_on() will catch if
  1278. * there is a race. We just do loose check here.
  1279. */
  1280. if (atomic_read(&memcg_drain_count))
  1281. return;
  1282. /* Notify other cpus that system-wide "drain" is running */
  1283. atomic_inc(&memcg_drain_count);
  1284. get_online_cpus();
  1285. for_each_online_cpu(cpu) {
  1286. struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
  1287. schedule_work_on(cpu, &stock->work);
  1288. }
  1289. put_online_cpus();
  1290. atomic_dec(&memcg_drain_count);
  1291. /* We don't wait for flush_work */
  1292. }
  1293. /* This is a synchronous drain interface. */
  1294. static void drain_all_stock_sync(void)
  1295. {
  1296. /* called when force_empty is called */
  1297. atomic_inc(&memcg_drain_count);
  1298. schedule_on_each_cpu(drain_local_stock);
  1299. atomic_dec(&memcg_drain_count);
  1300. }
  1301. static int __cpuinit memcg_stock_cpu_callback(struct notifier_block *nb,
  1302. unsigned long action,
  1303. void *hcpu)
  1304. {
  1305. int cpu = (unsigned long)hcpu;
  1306. struct memcg_stock_pcp *stock;
  1307. if (action != CPU_DEAD)
  1308. return NOTIFY_OK;
  1309. stock = &per_cpu(memcg_stock, cpu);
  1310. drain_stock(stock);
  1311. return NOTIFY_OK;
  1312. }
  1313. /*
  1314. * Unlike exported interface, "oom" parameter is added. if oom==true,
  1315. * oom-killer can be invoked.
  1316. */
  1317. static int __mem_cgroup_try_charge(struct mm_struct *mm,
  1318. gfp_t gfp_mask, struct mem_cgroup **memcg, bool oom)
  1319. {
  1320. struct mem_cgroup *mem, *mem_over_limit;
  1321. int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
  1322. struct res_counter *fail_res;
  1323. int csize = CHARGE_SIZE;
  1324. /*
  1325. * Unlike gloval-vm's OOM-kill, we're not in memory shortage
  1326. * in system level. So, allow to go ahead dying process in addition to
  1327. * MEMDIE process.
  1328. */
  1329. if (unlikely(test_thread_flag(TIF_MEMDIE)
  1330. || fatal_signal_pending(current)))
  1331. goto bypass;
  1332. /*
  1333. * We always charge the cgroup the mm_struct belongs to.
  1334. * The mm_struct's mem_cgroup changes on task migration if the
  1335. * thread group leader migrates. It's possible that mm is not
  1336. * set, if so charge the init_mm (happens for pagecache usage).
  1337. */
  1338. mem = *memcg;
  1339. if (likely(!mem)) {
  1340. mem = try_get_mem_cgroup_from_mm(mm);
  1341. *memcg = mem;
  1342. } else {
  1343. css_get(&mem->css);
  1344. }
  1345. if (unlikely(!mem))
  1346. return 0;
  1347. VM_BUG_ON(css_is_removed(&mem->css));
  1348. if (mem_cgroup_is_root(mem))
  1349. goto done;
  1350. while (1) {
  1351. int ret = 0;
  1352. unsigned long flags = 0;
  1353. if (consume_stock(mem))
  1354. goto done;
  1355. ret = res_counter_charge(&mem->res, csize, &fail_res);
  1356. if (likely(!ret)) {
  1357. if (!do_swap_account)
  1358. break;
  1359. ret = res_counter_charge(&mem->memsw, csize, &fail_res);
  1360. if (likely(!ret))
  1361. break;
  1362. /* mem+swap counter fails */
  1363. res_counter_uncharge(&mem->res, csize);
  1364. flags |= MEM_CGROUP_RECLAIM_NOSWAP;
  1365. mem_over_limit = mem_cgroup_from_res_counter(fail_res,
  1366. memsw);
  1367. } else
  1368. /* mem counter fails */
  1369. mem_over_limit = mem_cgroup_from_res_counter(fail_res,
  1370. res);
  1371. /* reduce request size and retry */
  1372. if (csize > PAGE_SIZE) {
  1373. csize = PAGE_SIZE;
  1374. continue;
  1375. }
  1376. if (!(gfp_mask & __GFP_WAIT))
  1377. goto nomem;
  1378. ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, NULL,
  1379. gfp_mask, flags);
  1380. if (ret)
  1381. continue;
  1382. /*
  1383. * try_to_free_mem_cgroup_pages() might not give us a full
  1384. * picture of reclaim. Some pages are reclaimed and might be
  1385. * moved to swap cache or just unmapped from the cgroup.
  1386. * Check the limit again to see if the reclaim reduced the
  1387. * current usage of the cgroup before giving up
  1388. *
  1389. */
  1390. if (mem_cgroup_check_under_limit(mem_over_limit))
  1391. continue;
  1392. /* try to avoid oom while someone is moving charge */
  1393. if (mc.moving_task && current != mc.moving_task) {
  1394. struct mem_cgroup *from, *to;
  1395. bool do_continue = false;
  1396. /*
  1397. * There is a small race that "from" or "to" can be
  1398. * freed by rmdir, so we use css_tryget().
  1399. */
  1400. rcu_read_lock();
  1401. from = mc.from;
  1402. to = mc.to;
  1403. if (from && css_tryget(&from->css)) {
  1404. if (mem_over_limit->use_hierarchy)
  1405. do_continue = css_is_ancestor(
  1406. &from->css,
  1407. &mem_over_limit->css);
  1408. else
  1409. do_continue = (from == mem_over_limit);
  1410. css_put(&from->css);
  1411. }
  1412. if (!do_continue && to && css_tryget(&to->css)) {
  1413. if (mem_over_limit->use_hierarchy)
  1414. do_continue = css_is_ancestor(
  1415. &to->css,
  1416. &mem_over_limit->css);
  1417. else
  1418. do_continue = (to == mem_over_limit);
  1419. css_put(&to->css);
  1420. }
  1421. rcu_read_unlock();
  1422. if (do_continue) {
  1423. DEFINE_WAIT(wait);
  1424. prepare_to_wait(&mc.waitq, &wait,
  1425. TASK_INTERRUPTIBLE);
  1426. /* moving charge context might have finished. */
  1427. if (mc.moving_task)
  1428. schedule();
  1429. finish_wait(&mc.waitq, &wait);
  1430. continue;
  1431. }
  1432. }
  1433. if (!nr_retries--) {
  1434. if (!oom)
  1435. goto nomem;
  1436. if (mem_cgroup_handle_oom(mem_over_limit, gfp_mask)) {
  1437. nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
  1438. continue;
  1439. }
  1440. /* When we reach here, current task is dying .*/
  1441. css_put(&mem->css);
  1442. goto bypass;
  1443. }
  1444. }
  1445. if (csize > PAGE_SIZE)
  1446. refill_stock(mem, csize - PAGE_SIZE);
  1447. done:
  1448. return 0;
  1449. nomem:
  1450. css_put(&mem->css);
  1451. return -ENOMEM;
  1452. bypass:
  1453. *memcg = NULL;
  1454. return 0;
  1455. }
  1456. /*
  1457. * Somemtimes we have to undo a charge we got by try_charge().
  1458. * This function is for that and do uncharge, put css's refcnt.
  1459. * gotten by try_charge().
  1460. */
  1461. static void __mem_cgroup_cancel_charge(struct mem_cgroup *mem,
  1462. unsigned long count)
  1463. {
  1464. if (!mem_cgroup_is_root(mem)) {
  1465. res_counter_uncharge(&mem->res, PAGE_SIZE * count);
  1466. if (do_swap_account)
  1467. res_counter_uncharge(&mem->memsw, PAGE_SIZE * count);
  1468. VM_BUG_ON(test_bit(CSS_ROOT, &mem->css.flags));
  1469. WARN_ON_ONCE(count > INT_MAX);
  1470. __css_put(&mem->css, (int)count);
  1471. }
  1472. /* we don't need css_put for root */
  1473. }
  1474. static void mem_cgroup_cancel_charge(struct mem_cgroup *mem)
  1475. {
  1476. __mem_cgroup_cancel_charge(mem, 1);
  1477. }
  1478. /*
  1479. * A helper function to get mem_cgroup from ID. must be called under
  1480. * rcu_read_lock(). The caller must check css_is_removed() or some if
  1481. * it's concern. (dropping refcnt from swap can be called against removed
  1482. * memcg.)
  1483. */
  1484. static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
  1485. {
  1486. struct cgroup_subsys_state *css;
  1487. /* ID 0 is unused ID */
  1488. if (!id)
  1489. return NULL;
  1490. css = css_lookup(&mem_cgroup_subsys, id);
  1491. if (!css)
  1492. return NULL;
  1493. return container_of(css, struct mem_cgroup, css);
  1494. }
  1495. struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
  1496. {
  1497. struct mem_cgroup *mem = NULL;
  1498. struct page_cgroup *pc;
  1499. unsigned short id;
  1500. swp_entry_t ent;
  1501. VM_BUG_ON(!PageLocked(page));
  1502. pc = lookup_page_cgroup(page);
  1503. lock_page_cgroup(pc);
  1504. if (PageCgroupUsed(pc)) {
  1505. mem = pc->mem_cgroup;
  1506. if (mem && !css_tryget(&mem->css))
  1507. mem = NULL;
  1508. } else if (PageSwapCache(page)) {
  1509. ent.val = page_private(page);
  1510. id = lookup_swap_cgroup(ent);
  1511. rcu_read_lock();
  1512. mem = mem_cgroup_lookup(id);
  1513. if (mem && !css_tryget(&mem->css))
  1514. mem = NULL;
  1515. rcu_read_unlock();
  1516. }
  1517. unlock_page_cgroup(pc);
  1518. return mem;
  1519. }
  1520. /*
  1521. * commit a charge got by __mem_cgroup_try_charge() and makes page_cgroup to be
  1522. * USED state. If already USED, uncharge and return.
  1523. */
  1524. static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
  1525. struct page_cgroup *pc,
  1526. enum charge_type ctype)
  1527. {
  1528. /* try_charge() can return NULL to *memcg, taking care of it. */
  1529. if (!mem)
  1530. return;
  1531. lock_page_cgroup(pc);
  1532. if (unlikely(PageCgroupUsed(pc))) {
  1533. unlock_page_cgroup(pc);
  1534. mem_cgroup_cancel_charge(mem);
  1535. return;
  1536. }
  1537. pc->mem_cgroup = mem;
  1538. /*
  1539. * We access a page_cgroup asynchronously without lock_page_cgroup().
  1540. * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
  1541. * is accessed after testing USED bit. To make pc->mem_cgroup visible
  1542. * before USED bit, we need memory barrier here.
  1543. * See mem_cgroup_add_lru_list(), etc.
  1544. */
  1545. smp_wmb();
  1546. switch (ctype) {
  1547. case MEM_CGROUP_CHARGE_TYPE_CACHE:
  1548. case MEM_CGROUP_CHARGE_TYPE_SHMEM:
  1549. SetPageCgroupCache(pc);
  1550. SetPageCgroupUsed(pc);
  1551. break;
  1552. case MEM_CGROUP_CHARGE_TYPE_MAPPED:
  1553. ClearPageCgroupCache(pc);
  1554. SetPageCgroupUsed(pc);
  1555. break;
  1556. default:
  1557. break;
  1558. }
  1559. mem_cgroup_charge_statistics(mem, pc, true);
  1560. unlock_page_cgroup(pc);
  1561. /*
  1562. * "charge_statistics" updated event counter. Then, check it.
  1563. * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
  1564. * if they exceeds softlimit.
  1565. */
  1566. memcg_check_events(mem, pc->page);
  1567. }
  1568. /**
  1569. * __mem_cgroup_move_account - move account of the page
  1570. * @pc: page_cgroup of the page.
  1571. * @from: mem_cgroup which the page is moved from.
  1572. * @to: mem_cgroup which the page is moved to. @from != @to.
  1573. * @uncharge: whether we should call uncharge and css_put against @from.
  1574. *
  1575. * The caller must confirm following.
  1576. * - page is not on LRU (isolate_page() is useful.)
  1577. * - the pc is locked, used, and ->mem_cgroup points to @from.
  1578. *
  1579. * This function doesn't do "charge" nor css_get to new cgroup. It should be
  1580. * done by a caller(__mem_cgroup_try_charge would be usefull). If @uncharge is
  1581. * true, this function does "uncharge" from old cgroup, but it doesn't if
  1582. * @uncharge is false, so a caller should do "uncharge".
  1583. */
  1584. static void __mem_cgroup_move_account(struct page_cgroup *pc,
  1585. struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge)
  1586. {
  1587. VM_BUG_ON(from == to);
  1588. VM_BUG_ON(PageLRU(pc->page));
  1589. VM_BUG_ON(!PageCgroupLocked(pc));
  1590. VM_BUG_ON(!PageCgroupUsed(pc));
  1591. VM_BUG_ON(pc->mem_cgroup != from);
  1592. if (PageCgroupFileMapped(pc)) {
  1593. /* Update mapped_file data for mem_cgroup */
  1594. preempt_disable();
  1595. __this_cpu_dec(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
  1596. __this_cpu_inc(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
  1597. preempt_enable();
  1598. }
  1599. mem_cgroup_charge_statistics(from, pc, false);
  1600. if (uncharge)
  1601. /* This is not "cancel", but cancel_charge does all we need. */
  1602. mem_cgroup_cancel_charge(from);
  1603. /* caller should have done css_get */
  1604. pc->mem_cgroup = to;
  1605. mem_cgroup_charge_statistics(to, pc, true);
  1606. /*
  1607. * We charges against "to" which may not have any tasks. Then, "to"
  1608. * can be under rmdir(). But in current implementation, caller of
  1609. * this function is just force_empty() and move charge, so it's
  1610. * garanteed that "to" is never removed. So, we don't check rmdir
  1611. * status here.
  1612. */
  1613. }
  1614. /*
  1615. * check whether the @pc is valid for moving account and call
  1616. * __mem_cgroup_move_account()
  1617. */
  1618. static int mem_cgroup_move_account(struct page_cgroup *pc,
  1619. struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge)
  1620. {
  1621. int ret = -EINVAL;
  1622. lock_page_cgroup(pc);
  1623. if (PageCgroupUsed(pc) && pc->mem_cgroup == from) {
  1624. __mem_cgroup_move_account(pc, from, to, uncharge);
  1625. ret = 0;
  1626. }
  1627. unlock_page_cgroup(pc);
  1628. /*
  1629. * check events
  1630. */
  1631. memcg_check_events(to, pc->page);
  1632. memcg_check_events(from, pc->page);
  1633. return ret;
  1634. }
  1635. /*
  1636. * move charges to its parent.
  1637. */
  1638. static int mem_cgroup_move_parent(struct page_cgroup *pc,
  1639. struct mem_cgroup *child,
  1640. gfp_t gfp_mask)
  1641. {
  1642. struct page *page = pc->page;
  1643. struct cgroup *cg = child->css.cgroup;
  1644. struct cgroup *pcg = cg->parent;
  1645. struct mem_cgroup *parent;
  1646. int ret;
  1647. /* Is ROOT ? */
  1648. if (!pcg)
  1649. return -EINVAL;
  1650. ret = -EBUSY;
  1651. if (!get_page_unless_zero(page))
  1652. goto out;
  1653. if (isolate_lru_page(page))
  1654. goto put;
  1655. parent = mem_cgroup_from_cont(pcg);
  1656. ret = __mem_cgroup_try_charge(NULL, gfp_mask, &parent, false);
  1657. if (ret || !parent)
  1658. goto put_back;
  1659. ret = mem_cgroup_move_account(pc, child, parent, true);
  1660. if (ret)
  1661. mem_cgroup_cancel_charge(parent);
  1662. put_back:
  1663. putback_lru_page(page);
  1664. put:
  1665. put_page(page);
  1666. out:
  1667. return ret;
  1668. }
  1669. /*
  1670. * Charge the memory controller for page usage.
  1671. * Return
  1672. * 0 if the charge was successful
  1673. * < 0 if the cgroup is over its limit
  1674. */
  1675. static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
  1676. gfp_t gfp_mask, enum charge_type ctype,
  1677. struct mem_cgroup *memcg)
  1678. {
  1679. struct mem_cgroup *mem;
  1680. struct page_cgroup *pc;
  1681. int ret;
  1682. pc = lookup_page_cgroup(page);
  1683. /* can happen at boot */
  1684. if (unlikely(!pc))
  1685. return 0;
  1686. prefetchw(pc);
  1687. mem = memcg;
  1688. ret = __mem_cgroup_try_charge(mm, gfp_mask, &mem, true);
  1689. if (ret || !mem)
  1690. return ret;
  1691. __mem_cgroup_commit_charge(mem, pc, ctype);
  1692. return 0;
  1693. }
  1694. int mem_cgroup_newpage_charge(struct page *page,
  1695. struct mm_struct *mm, gfp_t gfp_mask)
  1696. {
  1697. if (mem_cgroup_disabled())
  1698. return 0;
  1699. if (PageCompound(page))
  1700. return 0;
  1701. /*
  1702. * If already mapped, we don't have to account.
  1703. * If page cache, page->mapping has address_space.
  1704. * But page->mapping may have out-of-use anon_vma pointer,
  1705. * detecit it by PageAnon() check. newly-mapped-anon's page->mapping
  1706. * is NULL.
  1707. */
  1708. if (page_mapped(page) || (page->mapping && !PageAnon(page)))
  1709. return 0;
  1710. if (unlikely(!mm))
  1711. mm = &init_mm;
  1712. return mem_cgroup_charge_common(page, mm, gfp_mask,
  1713. MEM_CGROUP_CHARGE_TYPE_MAPPED, NULL);
  1714. }
  1715. static void
  1716. __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
  1717. enum charge_type ctype);
  1718. int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
  1719. gfp_t gfp_mask)
  1720. {
  1721. struct mem_cgroup *mem = NULL;
  1722. int ret;
  1723. if (mem_cgroup_disabled())
  1724. return 0;
  1725. if (PageCompound(page))
  1726. return 0;
  1727. /*
  1728. * Corner case handling. This is called from add_to_page_cache()
  1729. * in usual. But some FS (shmem) precharges this page before calling it
  1730. * and call add_to_page_cache() with GFP_NOWAIT.
  1731. *
  1732. * For GFP_NOWAIT case, the page may be pre-charged before calling
  1733. * add_to_page_cache(). (See shmem.c) check it here and avoid to call
  1734. * charge twice. (It works but has to pay a bit larger cost.)
  1735. * And when the page is SwapCache, it should take swap information
  1736. * into account. This is under lock_page() now.
  1737. */
  1738. if (!(gfp_mask & __GFP_WAIT)) {
  1739. struct page_cgroup *pc;
  1740. pc = lookup_page_cgroup(page);
  1741. if (!pc)
  1742. return 0;
  1743. lock_page_cgroup(pc);
  1744. if (PageCgroupUsed(pc)) {
  1745. unlock_page_cgroup(pc);
  1746. return 0;
  1747. }
  1748. unlock_page_cgroup(pc);
  1749. }
  1750. if (unlikely(!mm && !mem))
  1751. mm = &init_mm;
  1752. if (page_is_file_cache(page))
  1753. return mem_cgroup_charge_common(page, mm, gfp_mask,
  1754. MEM_CGROUP_CHARGE_TYPE_CACHE, NULL);
  1755. /* shmem */
  1756. if (PageSwapCache(page)) {
  1757. ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
  1758. if (!ret)
  1759. __mem_cgroup_commit_charge_swapin(page, mem,
  1760. MEM_CGROUP_CHARGE_TYPE_SHMEM);
  1761. } else
  1762. ret = mem_cgroup_charge_common(page, mm, gfp_mask,
  1763. MEM_CGROUP_CHARGE_TYPE_SHMEM, mem);
  1764. return ret;
  1765. }
  1766. /*
  1767. * While swap-in, try_charge -> commit or cancel, the page is locked.
  1768. * And when try_charge() successfully returns, one refcnt to memcg without
  1769. * struct page_cgroup is acquired. This refcnt will be consumed by
  1770. * "commit()" or removed by "cancel()"
  1771. */
  1772. int mem_cgroup_try_charge_swapin(struct mm_struct *mm,
  1773. struct page *page,
  1774. gfp_t mask, struct mem_cgroup **ptr)
  1775. {
  1776. struct mem_cgroup *mem;
  1777. int ret;
  1778. if (mem_cgroup_disabled())
  1779. return 0;
  1780. if (!do_swap_account)
  1781. goto charge_cur_mm;
  1782. /*
  1783. * A racing thread's fault, or swapoff, may have already updated
  1784. * the pte, and even removed page from swap cache: in those cases
  1785. * do_swap_page()'s pte_same() test will fail; but there's also a
  1786. * KSM case which does need to charge the page.
  1787. */
  1788. if (!PageSwapCache(page))
  1789. goto charge_cur_mm;
  1790. mem = try_get_mem_cgroup_from_page(page);
  1791. if (!mem)
  1792. goto charge_cur_mm;
  1793. *ptr = mem;
  1794. ret = __mem_cgroup_try_charge(NULL, mask, ptr, true);
  1795. /* drop extra refcnt from tryget */
  1796. css_put(&mem->css);
  1797. return ret;
  1798. charge_cur_mm:
  1799. if (unlikely(!mm))
  1800. mm = &init_mm;
  1801. return __mem_cgroup_try_charge(mm, mask, ptr, true);
  1802. }
  1803. static void
  1804. __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
  1805. enum charge_type ctype)
  1806. {
  1807. struct page_cgroup *pc;
  1808. if (mem_cgroup_disabled())
  1809. return;
  1810. if (!ptr)
  1811. return;
  1812. cgroup_exclude_rmdir(&ptr->css);
  1813. pc = lookup_page_cgroup(page);
  1814. mem_cgroup_lru_del_before_commit_swapcache(page);
  1815. __mem_cgroup_commit_charge(ptr, pc, ctype);
  1816. mem_cgroup_lru_add_after_commit_swapcache(page);
  1817. /*
  1818. * Now swap is on-memory. This means this page may be
  1819. * counted both as mem and swap....double count.
  1820. * Fix it by uncharging from memsw. Basically, this SwapCache is stable
  1821. * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
  1822. * may call delete_from_swap_cache() before reach here.
  1823. */
  1824. if (do_swap_account && PageSwapCache(page)) {
  1825. swp_entry_t ent = {.val = page_private(page)};
  1826. unsigned short id;
  1827. struct mem_cgroup *memcg;
  1828. id = swap_cgroup_record(ent, 0);
  1829. rcu_read_lock();
  1830. memcg = mem_cgroup_lookup(id);
  1831. if (memcg) {
  1832. /*
  1833. * This recorded memcg can be obsolete one. So, avoid
  1834. * calling css_tryget
  1835. */
  1836. if (!mem_cgroup_is_root(memcg))
  1837. res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
  1838. mem_cgroup_swap_statistics(memcg, false);
  1839. mem_cgroup_put(memcg);
  1840. }
  1841. rcu_read_unlock();
  1842. }
  1843. /*
  1844. * At swapin, we may charge account against cgroup which has no tasks.
  1845. * So, rmdir()->pre_destroy() can be called while we do this charge.
  1846. * In that case, we need to call pre_destroy() again. check it here.
  1847. */
  1848. cgroup_release_and_wakeup_rmdir(&ptr->css);
  1849. }
  1850. void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr)
  1851. {
  1852. __mem_cgroup_commit_charge_swapin(page, ptr,
  1853. MEM_CGROUP_CHARGE_TYPE_MAPPED);
  1854. }
  1855. void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem)
  1856. {
  1857. if (mem_cgroup_disabled())
  1858. return;
  1859. if (!mem)
  1860. return;
  1861. mem_cgroup_cancel_charge(mem);
  1862. }
  1863. static void
  1864. __do_uncharge(struct mem_cgroup *mem, const enum charge_type ctype)
  1865. {
  1866. struct memcg_batch_info *batch = NULL;
  1867. bool uncharge_memsw = true;
  1868. /* If swapout, usage of swap doesn't decrease */
  1869. if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
  1870. uncharge_memsw = false;
  1871. /*
  1872. * do_batch > 0 when unmapping pages or inode invalidate/truncate.
  1873. * In those cases, all pages freed continously can be expected to be in
  1874. * the same cgroup and we have chance to coalesce uncharges.
  1875. * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE)
  1876. * because we want to do uncharge as soon as possible.
  1877. */
  1878. if (!current->memcg_batch.do_batch || test_thread_flag(TIF_MEMDIE))
  1879. goto direct_uncharge;
  1880. batch = &current->memcg_batch;
  1881. /*
  1882. * In usual, we do css_get() when we remember memcg pointer.
  1883. * But in this case, we keep res->usage until end of a series of
  1884. * uncharges. Then, it's ok to ignore memcg's refcnt.
  1885. */
  1886. if (!batch->memcg)
  1887. batch->memcg = mem;
  1888. /*
  1889. * In typical case, batch->memcg == mem. This means we can
  1890. * merge a series of uncharges to an uncharge of res_counter.
  1891. * If not, we uncharge res_counter ony by one.
  1892. */
  1893. if (batch->memcg != mem)
  1894. goto direct_uncharge;
  1895. /* remember freed charge and uncharge it later */
  1896. batch->bytes += PAGE_SIZE;
  1897. if (uncharge_memsw)
  1898. batch->memsw_bytes += PAGE_SIZE;
  1899. return;
  1900. direct_uncharge:
  1901. res_counter_uncharge(&mem->res, PAGE_SIZE);
  1902. if (uncharge_memsw)
  1903. res_counter_uncharge(&mem->memsw, PAGE_SIZE);
  1904. return;
  1905. }
  1906. /*
  1907. * uncharge if !page_mapped(page)
  1908. */
  1909. static struct mem_cgroup *
  1910. __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
  1911. {
  1912. struct page_cgroup *pc;
  1913. struct mem_cgroup *mem = NULL;
  1914. struct mem_cgroup_per_zone *mz;
  1915. if (mem_cgroup_disabled())
  1916. return NULL;
  1917. if (PageSwapCache(page))
  1918. return NULL;
  1919. /*
  1920. * Check if our page_cgroup is valid
  1921. */
  1922. pc = lookup_page_cgroup(page);
  1923. if (unlikely(!pc || !PageCgroupUsed(pc)))
  1924. return NULL;
  1925. lock_page_cgroup(pc);
  1926. mem = pc->mem_cgroup;
  1927. if (!PageCgroupUsed(pc))
  1928. goto unlock_out;
  1929. switch (ctype) {
  1930. case MEM_CGROUP_CHARGE_TYPE_MAPPED:
  1931. case MEM_CGROUP_CHARGE_TYPE_DROP:
  1932. if (page_mapped(page))
  1933. goto unlock_out;
  1934. break;
  1935. case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
  1936. if (!PageAnon(page)) { /* Shared memory */
  1937. if (page->mapping && !page_is_file_cache(page))
  1938. goto unlock_out;
  1939. } else if (page_mapped(page)) /* Anon */
  1940. goto unlock_out;
  1941. break;
  1942. default:
  1943. break;
  1944. }
  1945. if (!mem_cgroup_is_root(mem))
  1946. __do_uncharge(mem, ctype);
  1947. if (ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
  1948. mem_cgroup_swap_statistics(mem, true);
  1949. mem_cgroup_charge_statistics(mem, pc, false);
  1950. ClearPageCgroupUsed(pc);
  1951. /*
  1952. * pc->mem_cgroup is not cleared here. It will be accessed when it's
  1953. * freed from LRU. This is safe because uncharged page is expected not
  1954. * to be reused (freed soon). Exception is SwapCache, it's handled by
  1955. * special functions.
  1956. */
  1957. mz = page_cgroup_zoneinfo(pc);
  1958. unlock_page_cgroup(pc);
  1959. memcg_check_events(mem, page);
  1960. /* at swapout, this memcg will be accessed to record to swap */
  1961. if (ctype != MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
  1962. css_put(&mem->css);
  1963. return mem;
  1964. unlock_out:
  1965. unlock_page_cgroup(pc);
  1966. return NULL;
  1967. }
  1968. void mem_cgroup_uncharge_page(struct page *page)
  1969. {
  1970. /* early check. */
  1971. if (page_mapped(page))
  1972. return;
  1973. if (page->mapping && !PageAnon(page))
  1974. return;
  1975. __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED);
  1976. }
  1977. void mem_cgroup_uncharge_cache_page(struct page *page)
  1978. {
  1979. VM_BUG_ON(page_mapped(page));
  1980. VM_BUG_ON(page->mapping);
  1981. __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE);
  1982. }
  1983. /*
  1984. * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate.
  1985. * In that cases, pages are freed continuously and we can expect pages
  1986. * are in the same memcg. All these calls itself limits the number of
  1987. * pages freed at once, then uncharge_start/end() is called properly.
  1988. * This may be called prural(2) times in a context,
  1989. */
  1990. void mem_cgroup_uncharge_start(void)
  1991. {
  1992. current->memcg_batch.do_batch++;
  1993. /* We can do nest. */
  1994. if (current->memcg_batch.do_batch == 1) {
  1995. current->memcg_batch.memcg = NULL;
  1996. current->memcg_batch.bytes = 0;
  1997. current->memcg_batch.memsw_bytes = 0;
  1998. }
  1999. }
  2000. void mem_cgroup_uncharge_end(void)
  2001. {
  2002. struct memcg_batch_info *batch = &current->memcg_batch;
  2003. if (!batch->do_batch)
  2004. return;
  2005. batch->do_batch--;
  2006. if (batch->do_batch) /* If stacked, do nothing. */
  2007. return;
  2008. if (!batch->memcg)
  2009. return;
  2010. /*
  2011. * This "batch->memcg" is valid without any css_get/put etc...
  2012. * bacause we hide charges behind us.
  2013. */
  2014. if (batch->bytes)
  2015. res_counter_uncharge(&batch->memcg->res, batch->bytes);
  2016. if (batch->memsw_bytes)
  2017. res_counter_uncharge(&batch->memcg->memsw, batch->memsw_bytes);
  2018. /* forget this pointer (for sanity check) */
  2019. batch->memcg = NULL;
  2020. }
  2021. #ifdef CONFIG_SWAP
  2022. /*
  2023. * called after __delete_from_swap_cache() and drop "page" account.
  2024. * memcg information is recorded to swap_cgroup of "ent"
  2025. */
  2026. void
  2027. mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
  2028. {
  2029. struct mem_cgroup *memcg;
  2030. int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT;
  2031. if (!swapout) /* this was a swap cache but the swap is unused ! */
  2032. ctype = MEM_CGROUP_CHARGE_TYPE_DROP;
  2033. memcg = __mem_cgroup_uncharge_common(page, ctype);
  2034. /* record memcg information */
  2035. if (do_swap_account && swapout && memcg) {
  2036. rcu_read_lock();
  2037. swap_cgroup_record(ent, css_id(&memcg->css));
  2038. rcu_read_unlock();
  2039. mem_cgroup_get(memcg);
  2040. }
  2041. if (swapout && memcg)
  2042. css_put(&memcg->css);
  2043. }
  2044. #endif
  2045. #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
  2046. /*
  2047. * called from swap_entry_free(). remove record in swap_cgroup and
  2048. * uncharge "memsw" account.
  2049. */
  2050. void mem_cgroup_uncharge_swap(swp_entry_t ent)
  2051. {
  2052. struct mem_cgroup *memcg;
  2053. unsigned short id;
  2054. if (!do_swap_account)
  2055. return;
  2056. id = swap_cgroup_record(ent, 0);
  2057. rcu_read_lock();
  2058. memcg = mem_cgroup_lookup(id);
  2059. if (memcg) {
  2060. /*
  2061. * We uncharge this because swap is freed.
  2062. * This memcg can be obsolete one. We avoid calling css_tryget
  2063. */
  2064. if (!mem_cgroup_is_root(memcg))
  2065. res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
  2066. mem_cgroup_swap_statistics(memcg, false);
  2067. mem_cgroup_put(memcg);
  2068. }
  2069. rcu_read_unlock();
  2070. }
  2071. /**
  2072. * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
  2073. * @entry: swap entry to be moved
  2074. * @from: mem_cgroup which the entry is moved from
  2075. * @to: mem_cgroup which the entry is moved to
  2076. * @need_fixup: whether we should fixup res_counters and refcounts.
  2077. *
  2078. * It succeeds only when the swap_cgroup's record for this entry is the same
  2079. * as the mem_cgroup's id of @from.
  2080. *
  2081. * Returns 0 on success, -EINVAL on failure.
  2082. *
  2083. * The caller must have charged to @to, IOW, called res_counter_charge() about
  2084. * both res and memsw, and called css_get().
  2085. */
  2086. static int mem_cgroup_move_swap_account(swp_entry_t entry,
  2087. struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup)
  2088. {
  2089. unsigned short old_id, new_id;
  2090. rcu_read_lock();
  2091. old_id = css_id(&from->css);
  2092. new_id = css_id(&to->css);
  2093. rcu_read_unlock();
  2094. if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
  2095. mem_cgroup_swap_statistics(from, false);
  2096. mem_cgroup_swap_statistics(to, true);
  2097. /*
  2098. * This function is only called from task migration context now.
  2099. * It postpones res_counter and refcount handling till the end
  2100. * of task migration(mem_cgroup_clear_mc()) for performance
  2101. * improvement. But we cannot postpone mem_cgroup_get(to)
  2102. * because if the process that has been moved to @to does
  2103. * swap-in, the refcount of @to might be decreased to 0.
  2104. */
  2105. mem_cgroup_get(to);
  2106. if (need_fixup) {
  2107. if (!mem_cgroup_is_root(from))
  2108. res_counter_uncharge(&from->memsw, PAGE_SIZE);
  2109. mem_cgroup_put(from);
  2110. /*
  2111. * we charged both to->res and to->memsw, so we should
  2112. * uncharge to->res.
  2113. */
  2114. if (!mem_cgroup_is_root(to))
  2115. res_counter_uncharge(&to->res, PAGE_SIZE);
  2116. css_put(&to->css);
  2117. }
  2118. return 0;
  2119. }
  2120. return -EINVAL;
  2121. }
  2122. #else
  2123. static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
  2124. struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup)
  2125. {
  2126. return -EINVAL;
  2127. }
  2128. #endif
  2129. /*
  2130. * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
  2131. * page belongs to.
  2132. */
  2133. int mem_cgroup_prepare_migration(struct page *page, struct mem_cgroup **ptr)
  2134. {
  2135. struct page_cgroup *pc;
  2136. struct mem_cgroup *mem = NULL;
  2137. int ret = 0;
  2138. if (mem_cgroup_disabled())
  2139. return 0;
  2140. pc = lookup_page_cgroup(page);
  2141. lock_page_cgroup(pc);
  2142. if (PageCgroupUsed(pc)) {
  2143. mem = pc->mem_cgroup;
  2144. css_get(&mem->css);
  2145. }
  2146. unlock_page_cgroup(pc);
  2147. *ptr = mem;
  2148. if (mem) {
  2149. ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, ptr, false);
  2150. css_put(&mem->css);
  2151. }
  2152. return ret;
  2153. }
  2154. /* remove redundant charge if migration failed*/
  2155. void mem_cgroup_end_migration(struct mem_cgroup *mem,
  2156. struct page *oldpage, struct page *newpage)
  2157. {
  2158. struct page *target, *unused;
  2159. struct page_cgroup *pc;
  2160. enum charge_type ctype;
  2161. if (!mem)
  2162. return;
  2163. cgroup_exclude_rmdir(&mem->css);
  2164. /* at migration success, oldpage->mapping is NULL. */
  2165. if (oldpage->mapping) {
  2166. target = oldpage;
  2167. unused = NULL;
  2168. } else {
  2169. target = newpage;
  2170. unused = oldpage;
  2171. }
  2172. if (PageAnon(target))
  2173. ctype = MEM_CGROUP_CHARGE_TYPE_MAPPED;
  2174. else if (page_is_file_cache(target))
  2175. ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
  2176. else
  2177. ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM;
  2178. /* unused page is not on radix-tree now. */
  2179. if (unused)
  2180. __mem_cgroup_uncharge_common(unused, ctype);
  2181. pc = lookup_page_cgroup(target);
  2182. /*
  2183. * __mem_cgroup_commit_charge() check PCG_USED bit of page_cgroup.
  2184. * So, double-counting is effectively avoided.
  2185. */
  2186. __mem_cgroup_commit_charge(mem, pc, ctype);
  2187. /*
  2188. * Both of oldpage and newpage are still under lock_page().
  2189. * Then, we don't have to care about race in radix-tree.
  2190. * But we have to be careful that this page is unmapped or not.
  2191. *
  2192. * There is a case for !page_mapped(). At the start of
  2193. * migration, oldpage was mapped. But now, it's zapped.
  2194. * But we know *target* page is not freed/reused under us.
  2195. * mem_cgroup_uncharge_page() does all necessary checks.
  2196. */
  2197. if (ctype == MEM_CGROUP_CHARGE_TYPE_MAPPED)
  2198. mem_cgroup_uncharge_page(target);
  2199. /*
  2200. * At migration, we may charge account against cgroup which has no tasks
  2201. * So, rmdir()->pre_destroy() can be called while we do this charge.
  2202. * In that case, we need to call pre_destroy() again. check it here.
  2203. */
  2204. cgroup_release_and_wakeup_rmdir(&mem->css);
  2205. }
  2206. /*
  2207. * A call to try to shrink memory usage on charge failure at shmem's swapin.
  2208. * Calling hierarchical_reclaim is not enough because we should update
  2209. * last_oom_jiffies to prevent pagefault_out_of_memory from invoking global OOM.
  2210. * Moreover considering hierarchy, we should reclaim from the mem_over_limit,
  2211. * not from the memcg which this page would be charged to.
  2212. * try_charge_swapin does all of these works properly.
  2213. */
  2214. int mem_cgroup_shmem_charge_fallback(struct page *page,
  2215. struct mm_struct *mm,
  2216. gfp_t gfp_mask)
  2217. {
  2218. struct mem_cgroup *mem = NULL;
  2219. int ret;
  2220. if (mem_cgroup_disabled())
  2221. return 0;
  2222. ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
  2223. if (!ret)
  2224. mem_cgroup_cancel_charge_swapin(mem); /* it does !mem check */
  2225. return ret;
  2226. }
  2227. static DEFINE_MUTEX(set_limit_mutex);
  2228. static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
  2229. unsigned long long val)
  2230. {
  2231. int retry_count;
  2232. u64 memswlimit;
  2233. int ret = 0;
  2234. int children = mem_cgroup_count_children(memcg);
  2235. u64 curusage, oldusage;
  2236. /*
  2237. * For keeping hierarchical_reclaim simple, how long we should retry
  2238. * is depends on callers. We set our retry-count to be function
  2239. * of # of children which we should visit in this loop.
  2240. */
  2241. retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;
  2242. oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
  2243. while (retry_count) {
  2244. if (signal_pending(current)) {
  2245. ret = -EINTR;
  2246. break;
  2247. }
  2248. /*
  2249. * Rather than hide all in some function, I do this in
  2250. * open coded manner. You see what this really does.
  2251. * We have to guarantee mem->res.limit < mem->memsw.limit.
  2252. */
  2253. mutex_lock(&set_limit_mutex);
  2254. memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
  2255. if (memswlimit < val) {
  2256. ret = -EINVAL;
  2257. mutex_unlock(&set_limit_mutex);
  2258. break;
  2259. }
  2260. ret = res_counter_set_limit(&memcg->res, val);
  2261. if (!ret) {
  2262. if (memswlimit == val)
  2263. memcg->memsw_is_minimum = true;
  2264. else
  2265. memcg->memsw_is_minimum = false;
  2266. }
  2267. mutex_unlock(&set_limit_mutex);
  2268. if (!ret)
  2269. break;
  2270. mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
  2271. MEM_CGROUP_RECLAIM_SHRINK);
  2272. curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
  2273. /* Usage is reduced ? */
  2274. if (curusage >= oldusage)
  2275. retry_count--;
  2276. else
  2277. oldusage = curusage;
  2278. }
  2279. return ret;
  2280. }
  2281. static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
  2282. unsigned long long val)
  2283. {
  2284. int retry_count;
  2285. u64 memlimit, oldusage, curusage;
  2286. int children = mem_cgroup_count_children(memcg);
  2287. int ret = -EBUSY;
  2288. /* see mem_cgroup_resize_res_limit */
  2289. retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
  2290. oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
  2291. while (retry_count) {
  2292. if (signal_pending(current)) {
  2293. ret = -EINTR;
  2294. break;
  2295. }
  2296. /*
  2297. * Rather than hide all in some function, I do this in
  2298. * open coded manner. You see what this really does.
  2299. * We have to guarantee mem->res.limit < mem->memsw.limit.
  2300. */
  2301. mutex_lock(&set_limit_mutex);
  2302. memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
  2303. if (memlimit > val) {
  2304. ret = -EINVAL;
  2305. mutex_unlock(&set_limit_mutex);
  2306. break;
  2307. }
  2308. ret = res_counter_set_limit(&memcg->memsw, val);
  2309. if (!ret) {
  2310. if (memlimit == val)
  2311. memcg->memsw_is_minimum = true;
  2312. else
  2313. memcg->memsw_is_minimum = false;
  2314. }
  2315. mutex_unlock(&set_limit_mutex);
  2316. if (!ret)
  2317. break;
  2318. mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
  2319. MEM_CGROUP_RECLAIM_NOSWAP |
  2320. MEM_CGROUP_RECLAIM_SHRINK);
  2321. curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
  2322. /* Usage is reduced ? */
  2323. if (curusage >= oldusage)
  2324. retry_count--;
  2325. else
  2326. oldusage = curusage;
  2327. }
  2328. return ret;
  2329. }
  2330. unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
  2331. gfp_t gfp_mask, int nid,
  2332. int zid)
  2333. {
  2334. unsigned long nr_reclaimed = 0;
  2335. struct mem_cgroup_per_zone *mz, *next_mz = NULL;
  2336. unsigned long reclaimed;
  2337. int loop = 0;
  2338. struct mem_cgroup_tree_per_zone *mctz;
  2339. unsigned long long excess;
  2340. if (order > 0)
  2341. return 0;
  2342. mctz = soft_limit_tree_node_zone(nid, zid);
  2343. /*
  2344. * This loop can run a while, specially if mem_cgroup's continuously
  2345. * keep exceeding their soft limit and putting the system under
  2346. * pressure
  2347. */
  2348. do {
  2349. if (next_mz)
  2350. mz = next_mz;
  2351. else
  2352. mz = mem_cgroup_largest_soft_limit_node(mctz);
  2353. if (!mz)
  2354. break;
  2355. reclaimed = mem_cgroup_hierarchical_reclaim(mz->mem, zone,
  2356. gfp_mask,
  2357. MEM_CGROUP_RECLAIM_SOFT);
  2358. nr_reclaimed += reclaimed;
  2359. spin_lock(&mctz->lock);
  2360. /*
  2361. * If we failed to reclaim anything from this memory cgroup
  2362. * it is time to move on to the next cgroup
  2363. */
  2364. next_mz = NULL;
  2365. if (!reclaimed) {
  2366. do {
  2367. /*
  2368. * Loop until we find yet another one.
  2369. *
  2370. * By the time we get the soft_limit lock
  2371. * again, someone might have aded the
  2372. * group back on the RB tree. Iterate to
  2373. * make sure we get a different mem.
  2374. * mem_cgroup_largest_soft_limit_node returns
  2375. * NULL if no other cgroup is present on
  2376. * the tree
  2377. */
  2378. next_mz =
  2379. __mem_cgroup_largest_soft_limit_node(mctz);
  2380. if (next_mz == mz) {
  2381. css_put(&next_mz->mem->css);
  2382. next_mz = NULL;
  2383. } else /* next_mz == NULL or other memcg */
  2384. break;
  2385. } while (1);
  2386. }
  2387. __mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
  2388. excess = res_counter_soft_limit_excess(&mz->mem->res);
  2389. /*
  2390. * One school of thought says that we should not add
  2391. * back the node to the tree if reclaim returns 0.
  2392. * But our reclaim could return 0, simply because due
  2393. * to priority we are exposing a smaller subset of
  2394. * memory to reclaim from. Consider this as a longer
  2395. * term TODO.
  2396. */
  2397. /* If excess == 0, no tree ops */
  2398. __mem_cgroup_insert_exceeded(mz->mem, mz, mctz, excess);
  2399. spin_unlock(&mctz->lock);
  2400. css_put(&mz->mem->css);
  2401. loop++;
  2402. /*
  2403. * Could not reclaim anything and there are no more
  2404. * mem cgroups to try or we seem to be looping without
  2405. * reclaiming anything.
  2406. */
  2407. if (!nr_reclaimed &&
  2408. (next_mz == NULL ||
  2409. loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
  2410. break;
  2411. } while (!nr_reclaimed);
  2412. if (next_mz)
  2413. css_put(&next_mz->mem->css);
  2414. return nr_reclaimed;
  2415. }
  2416. /*
  2417. * This routine traverse page_cgroup in given list and drop them all.
  2418. * *And* this routine doesn't reclaim page itself, just removes page_cgroup.
  2419. */
  2420. static int mem_cgroup_force_empty_list(struct mem_cgroup *mem,
  2421. int node, int zid, enum lru_list lru)
  2422. {
  2423. struct zone *zone;
  2424. struct mem_cgroup_per_zone *mz;
  2425. struct page_cgroup *pc, *busy;
  2426. unsigned long flags, loop;
  2427. struct list_head *list;
  2428. int ret = 0;
  2429. zone = &NODE_DATA(node)->node_zones[zid];
  2430. mz = mem_cgroup_zoneinfo(mem, node, zid);
  2431. list = &mz->lists[lru];
  2432. loop = MEM_CGROUP_ZSTAT(mz, lru);
  2433. /* give some margin against EBUSY etc...*/
  2434. loop += 256;
  2435. busy = NULL;
  2436. while (loop--) {
  2437. ret = 0;
  2438. spin_lock_irqsave(&zone->lru_lock, flags);
  2439. if (list_empty(list)) {
  2440. spin_unlock_irqrestore(&zone->lru_lock, flags);
  2441. break;
  2442. }
  2443. pc = list_entry(list->prev, struct page_cgroup, lru);
  2444. if (busy == pc) {
  2445. list_move(&pc->lru, list);
  2446. busy = NULL;
  2447. spin_unlock_irqrestore(&zone->lru_lock, flags);
  2448. continue;
  2449. }
  2450. spin_unlock_irqrestore(&zone->lru_lock, flags);
  2451. ret = mem_cgroup_move_parent(pc, mem, GFP_KERNEL);
  2452. if (ret == -ENOMEM)
  2453. break;
  2454. if (ret == -EBUSY || ret == -EINVAL) {
  2455. /* found lock contention or "pc" is obsolete. */
  2456. busy = pc;
  2457. cond_resched();
  2458. } else
  2459. busy = NULL;
  2460. }
  2461. if (!ret && !list_empty(list))
  2462. return -EBUSY;
  2463. return ret;
  2464. }
  2465. /*
  2466. * make mem_cgroup's charge to be 0 if there is no task.
  2467. * This enables deleting this mem_cgroup.
  2468. */
  2469. static int mem_cgroup_force_empty(struct mem_cgroup *mem, bool free_all)
  2470. {
  2471. int ret;
  2472. int node, zid, shrink;
  2473. int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
  2474. struct cgroup *cgrp = mem->css.cgroup;
  2475. css_get(&mem->css);
  2476. shrink = 0;
  2477. /* should free all ? */
  2478. if (free_all)
  2479. goto try_to_free;
  2480. move_account:
  2481. do {
  2482. ret = -EBUSY;
  2483. if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
  2484. goto out;
  2485. ret = -EINTR;
  2486. if (signal_pending(current))
  2487. goto out;
  2488. /* This is for making all *used* pages to be on LRU. */
  2489. lru_add_drain_all();
  2490. drain_all_stock_sync();
  2491. ret = 0;
  2492. for_each_node_state(node, N_HIGH_MEMORY) {
  2493. for (zid = 0; !ret && zid < MAX_NR_ZONES; zid++) {
  2494. enum lru_list l;
  2495. for_each_lru(l) {
  2496. ret = mem_cgroup_force_empty_list(mem,
  2497. node, zid, l);
  2498. if (ret)
  2499. break;
  2500. }
  2501. }
  2502. if (ret)
  2503. break;
  2504. }
  2505. /* it seems parent cgroup doesn't have enough mem */
  2506. if (ret == -ENOMEM)
  2507. goto try_to_free;
  2508. cond_resched();
  2509. /* "ret" should also be checked to ensure all lists are empty. */
  2510. } while (mem->res.usage > 0 || ret);
  2511. out:
  2512. css_put(&mem->css);
  2513. return ret;
  2514. try_to_free:
  2515. /* returns EBUSY if there is a task or if we come here twice. */
  2516. if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children) || shrink) {
  2517. ret = -EBUSY;
  2518. goto out;
  2519. }
  2520. /* we call try-to-free pages for make this cgroup empty */
  2521. lru_add_drain_all();
  2522. /* try to free all pages in this cgroup */
  2523. shrink = 1;
  2524. while (nr_retries && mem->res.usage > 0) {
  2525. int progress;
  2526. if (signal_pending(current)) {
  2527. ret = -EINTR;
  2528. goto out;
  2529. }
  2530. progress = try_to_free_mem_cgroup_pages(mem, GFP_KERNEL,
  2531. false, get_swappiness(mem));
  2532. if (!progress) {
  2533. nr_retries--;
  2534. /* maybe some writeback is necessary */
  2535. congestion_wait(BLK_RW_ASYNC, HZ/10);
  2536. }
  2537. }
  2538. lru_add_drain();
  2539. /* try move_account...there may be some *locked* pages. */
  2540. goto move_account;
  2541. }
  2542. int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event)
  2543. {
  2544. return mem_cgroup_force_empty(mem_cgroup_from_cont(cont), true);
  2545. }
  2546. static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft)
  2547. {
  2548. return mem_cgroup_from_cont(cont)->use_hierarchy;
  2549. }
  2550. static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
  2551. u64 val)
  2552. {
  2553. int retval = 0;
  2554. struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
  2555. struct cgroup *parent = cont->parent;
  2556. struct mem_cgroup *parent_mem = NULL;
  2557. if (parent)
  2558. parent_mem = mem_cgroup_from_cont(parent);
  2559. cgroup_lock();
  2560. /*
  2561. * If parent's use_hierarchy is set, we can't make any modifications
  2562. * in the child subtrees. If it is unset, then the change can
  2563. * occur, provided the current cgroup has no children.
  2564. *
  2565. * For the root cgroup, parent_mem is NULL, we allow value to be
  2566. * set if there are no children.
  2567. */
  2568. if ((!parent_mem || !parent_mem->use_hierarchy) &&
  2569. (val == 1 || val == 0)) {
  2570. if (list_empty(&cont->children))
  2571. mem->use_hierarchy = val;
  2572. else
  2573. retval = -EBUSY;
  2574. } else
  2575. retval = -EINVAL;
  2576. cgroup_unlock();
  2577. return retval;
  2578. }
  2579. struct mem_cgroup_idx_data {
  2580. s64 val;
  2581. enum mem_cgroup_stat_index idx;
  2582. };
  2583. static int
  2584. mem_cgroup_get_idx_stat(struct mem_cgroup *mem, void *data)
  2585. {
  2586. struct mem_cgroup_idx_data *d = data;
  2587. d->val += mem_cgroup_read_stat(mem, d->idx);
  2588. return 0;
  2589. }
  2590. static void
  2591. mem_cgroup_get_recursive_idx_stat(struct mem_cgroup *mem,
  2592. enum mem_cgroup_stat_index idx, s64 *val)
  2593. {
  2594. struct mem_cgroup_idx_data d;
  2595. d.idx = idx;
  2596. d.val = 0;
  2597. mem_cgroup_walk_tree(mem, &d, mem_cgroup_get_idx_stat);
  2598. *val = d.val;
  2599. }
  2600. static inline u64 mem_cgroup_usage(struct mem_cgroup *mem, bool swap)
  2601. {
  2602. u64 idx_val, val;
  2603. if (!mem_cgroup_is_root(mem)) {
  2604. if (!swap)
  2605. return res_counter_read_u64(&mem->res, RES_USAGE);
  2606. else
  2607. return res_counter_read_u64(&mem->memsw, RES_USAGE);
  2608. }
  2609. mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_CACHE, &idx_val);
  2610. val = idx_val;
  2611. mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_RSS, &idx_val);
  2612. val += idx_val;
  2613. if (swap) {
  2614. mem_cgroup_get_recursive_idx_stat(mem,
  2615. MEM_CGROUP_STAT_SWAPOUT, &idx_val);
  2616. val += idx_val;
  2617. }
  2618. return val << PAGE_SHIFT;
  2619. }
  2620. static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft)
  2621. {
  2622. struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
  2623. u64 val;
  2624. int type, name;
  2625. type = MEMFILE_TYPE(cft->private);
  2626. name = MEMFILE_ATTR(cft->private);
  2627. switch (type) {
  2628. case _MEM:
  2629. if (name == RES_USAGE)
  2630. val = mem_cgroup_usage(mem, false);
  2631. else
  2632. val = res_counter_read_u64(&mem->res, name);
  2633. break;
  2634. case _MEMSWAP:
  2635. if (name == RES_USAGE)
  2636. val = mem_cgroup_usage(mem, true);
  2637. else
  2638. val = res_counter_read_u64(&mem->memsw, name);
  2639. break;
  2640. default:
  2641. BUG();
  2642. break;
  2643. }
  2644. return val;
  2645. }
  2646. /*
  2647. * The user of this function is...
  2648. * RES_LIMIT.
  2649. */
  2650. static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
  2651. const char *buffer)
  2652. {
  2653. struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
  2654. int type, name;
  2655. unsigned long long val;
  2656. int ret;
  2657. type = MEMFILE_TYPE(cft->private);
  2658. name = MEMFILE_ATTR(cft->private);
  2659. switch (name) {
  2660. case RES_LIMIT:
  2661. if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
  2662. ret = -EINVAL;
  2663. break;
  2664. }
  2665. /* This function does all necessary parse...reuse it */
  2666. ret = res_counter_memparse_write_strategy(buffer, &val);
  2667. if (ret)
  2668. break;
  2669. if (type == _MEM)
  2670. ret = mem_cgroup_resize_limit(memcg, val);
  2671. else
  2672. ret = mem_cgroup_resize_memsw_limit(memcg, val);
  2673. break;
  2674. case RES_SOFT_LIMIT:
  2675. ret = res_counter_memparse_write_strategy(buffer, &val);
  2676. if (ret)
  2677. break;
  2678. /*
  2679. * For memsw, soft limits are hard to implement in terms
  2680. * of semantics, for now, we support soft limits for
  2681. * control without swap
  2682. */
  2683. if (type == _MEM)
  2684. ret = res_counter_set_soft_limit(&memcg->res, val);
  2685. else
  2686. ret = -EINVAL;
  2687. break;
  2688. default:
  2689. ret = -EINVAL; /* should be BUG() ? */
  2690. break;
  2691. }
  2692. return ret;
  2693. }
  2694. static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg,
  2695. unsigned long long *mem_limit, unsigned long long *memsw_limit)
  2696. {
  2697. struct cgroup *cgroup;
  2698. unsigned long long min_limit, min_memsw_limit, tmp;
  2699. min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
  2700. min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
  2701. cgroup = memcg->css.cgroup;
  2702. if (!memcg->use_hierarchy)
  2703. goto out;
  2704. while (cgroup->parent) {
  2705. cgroup = cgroup->parent;
  2706. memcg = mem_cgroup_from_cont(cgroup);
  2707. if (!memcg->use_hierarchy)
  2708. break;
  2709. tmp = res_counter_read_u64(&memcg->res, RES_LIMIT);
  2710. min_limit = min(min_limit, tmp);
  2711. tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
  2712. min_memsw_limit = min(min_memsw_limit, tmp);
  2713. }
  2714. out:
  2715. *mem_limit = min_limit;
  2716. *memsw_limit = min_memsw_limit;
  2717. return;
  2718. }
  2719. static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
  2720. {
  2721. struct mem_cgroup *mem;
  2722. int type, name;
  2723. mem = mem_cgroup_from_cont(cont);
  2724. type = MEMFILE_TYPE(event);
  2725. name = MEMFILE_ATTR(event);
  2726. switch (name) {
  2727. case RES_MAX_USAGE:
  2728. if (type == _MEM)
  2729. res_counter_reset_max(&mem->res);
  2730. else
  2731. res_counter_reset_max(&mem->memsw);
  2732. break;
  2733. case RES_FAILCNT:
  2734. if (type == _MEM)
  2735. res_counter_reset_failcnt(&mem->res);
  2736. else
  2737. res_counter_reset_failcnt(&mem->memsw);
  2738. break;
  2739. }
  2740. return 0;
  2741. }
  2742. static u64 mem_cgroup_move_charge_read(struct cgroup *cgrp,
  2743. struct cftype *cft)
  2744. {
  2745. return mem_cgroup_from_cont(cgrp)->move_charge_at_immigrate;
  2746. }
  2747. #ifdef CONFIG_MMU
  2748. static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
  2749. struct cftype *cft, u64 val)
  2750. {
  2751. struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
  2752. if (val >= (1 << NR_MOVE_TYPE))
  2753. return -EINVAL;
  2754. /*
  2755. * We check this value several times in both in can_attach() and
  2756. * attach(), so we need cgroup lock to prevent this value from being
  2757. * inconsistent.
  2758. */
  2759. cgroup_lock();
  2760. mem->move_charge_at_immigrate = val;
  2761. cgroup_unlock();
  2762. return 0;
  2763. }
  2764. #else
  2765. static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
  2766. struct cftype *cft, u64 val)
  2767. {
  2768. return -ENOSYS;
  2769. }
  2770. #endif
  2771. /* For read statistics */
  2772. enum {
  2773. MCS_CACHE,
  2774. MCS_RSS,
  2775. MCS_FILE_MAPPED,
  2776. MCS_PGPGIN,
  2777. MCS_PGPGOUT,
  2778. MCS_SWAP,
  2779. MCS_INACTIVE_ANON,
  2780. MCS_ACTIVE_ANON,
  2781. MCS_INACTIVE_FILE,
  2782. MCS_ACTIVE_FILE,
  2783. MCS_UNEVICTABLE,
  2784. NR_MCS_STAT,
  2785. };
  2786. struct mcs_total_stat {
  2787. s64 stat[NR_MCS_STAT];
  2788. };
  2789. struct {
  2790. char *local_name;
  2791. char *total_name;
  2792. } memcg_stat_strings[NR_MCS_STAT] = {
  2793. {"cache", "total_cache"},
  2794. {"rss", "total_rss"},
  2795. {"mapped_file", "total_mapped_file"},
  2796. {"pgpgin", "total_pgpgin"},
  2797. {"pgpgout", "total_pgpgout"},
  2798. {"swap", "total_swap"},
  2799. {"inactive_anon", "total_inactive_anon"},
  2800. {"active_anon", "total_active_anon"},
  2801. {"inactive_file", "total_inactive_file"},
  2802. {"active_file", "total_active_file"},
  2803. {"unevictable", "total_unevictable"}
  2804. };
  2805. static int mem_cgroup_get_local_stat(struct mem_cgroup *mem, void *data)
  2806. {
  2807. struct mcs_total_stat *s = data;
  2808. s64 val;
  2809. /* per cpu stat */
  2810. val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE);
  2811. s->stat[MCS_CACHE] += val * PAGE_SIZE;
  2812. val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS);
  2813. s->stat[MCS_RSS] += val * PAGE_SIZE;
  2814. val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_FILE_MAPPED);
  2815. s->stat[MCS_FILE_MAPPED] += val * PAGE_SIZE;
  2816. val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGIN_COUNT);
  2817. s->stat[MCS_PGPGIN] += val;
  2818. val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGOUT_COUNT);
  2819. s->stat[MCS_PGPGOUT] += val;
  2820. if (do_swap_account) {
  2821. val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_SWAPOUT);
  2822. s->stat[MCS_SWAP] += val * PAGE_SIZE;
  2823. }
  2824. /* per zone stat */
  2825. val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_ANON);
  2826. s->stat[MCS_INACTIVE_ANON] += val * PAGE_SIZE;
  2827. val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_ANON);
  2828. s->stat[MCS_ACTIVE_ANON] += val * PAGE_SIZE;
  2829. val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_FILE);
  2830. s->stat[MCS_INACTIVE_FILE] += val * PAGE_SIZE;
  2831. val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_FILE);
  2832. s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE;
  2833. val = mem_cgroup_get_local_zonestat(mem, LRU_UNEVICTABLE);
  2834. s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE;
  2835. return 0;
  2836. }
  2837. static void
  2838. mem_cgroup_get_total_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
  2839. {
  2840. mem_cgroup_walk_tree(mem, s, mem_cgroup_get_local_stat);
  2841. }
  2842. static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
  2843. struct cgroup_map_cb *cb)
  2844. {
  2845. struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
  2846. struct mcs_total_stat mystat;
  2847. int i;
  2848. memset(&mystat, 0, sizeof(mystat));
  2849. mem_cgroup_get_local_stat(mem_cont, &mystat);
  2850. for (i = 0; i < NR_MCS_STAT; i++) {
  2851. if (i == MCS_SWAP && !do_swap_account)
  2852. continue;
  2853. cb->fill(cb, memcg_stat_strings[i].local_name, mystat.stat[i]);
  2854. }
  2855. /* Hierarchical information */
  2856. {
  2857. unsigned long long limit, memsw_limit;
  2858. memcg_get_hierarchical_limit(mem_cont, &limit, &memsw_limit);
  2859. cb->fill(cb, "hierarchical_memory_limit", limit);
  2860. if (do_swap_account)
  2861. cb->fill(cb, "hierarchical_memsw_limit", memsw_limit);
  2862. }
  2863. memset(&mystat, 0, sizeof(mystat));
  2864. mem_cgroup_get_total_stat(mem_cont, &mystat);
  2865. for (i = 0; i < NR_MCS_STAT; i++) {
  2866. if (i == MCS_SWAP && !do_swap_account)
  2867. continue;
  2868. cb->fill(cb, memcg_stat_strings[i].total_name, mystat.stat[i]);
  2869. }
  2870. #ifdef CONFIG_DEBUG_VM
  2871. cb->fill(cb, "inactive_ratio", calc_inactive_ratio(mem_cont, NULL));
  2872. {
  2873. int nid, zid;
  2874. struct mem_cgroup_per_zone *mz;
  2875. unsigned long recent_rotated[2] = {0, 0};
  2876. unsigned long recent_scanned[2] = {0, 0};
  2877. for_each_online_node(nid)
  2878. for (zid = 0; zid < MAX_NR_ZONES; zid++) {
  2879. mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
  2880. recent_rotated[0] +=
  2881. mz->reclaim_stat.recent_rotated[0];
  2882. recent_rotated[1] +=
  2883. mz->reclaim_stat.recent_rotated[1];
  2884. recent_scanned[0] +=
  2885. mz->reclaim_stat.recent_scanned[0];
  2886. recent_scanned[1] +=
  2887. mz->reclaim_stat.recent_scanned[1];
  2888. }
  2889. cb->fill(cb, "recent_rotated_anon", recent_rotated[0]);
  2890. cb->fill(cb, "recent_rotated_file", recent_rotated[1]);
  2891. cb->fill(cb, "recent_scanned_anon", recent_scanned[0]);
  2892. cb->fill(cb, "recent_scanned_file", recent_scanned[1]);
  2893. }
  2894. #endif
  2895. return 0;
  2896. }
  2897. static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft)
  2898. {
  2899. struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
  2900. return get_swappiness(memcg);
  2901. }
  2902. static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft,
  2903. u64 val)
  2904. {
  2905. struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
  2906. struct mem_cgroup *parent;
  2907. if (val > 100)
  2908. return -EINVAL;
  2909. if (cgrp->parent == NULL)
  2910. return -EINVAL;
  2911. parent = mem_cgroup_from_cont(cgrp->parent);
  2912. cgroup_lock();
  2913. /* If under hierarchy, only empty-root can set this value */
  2914. if ((parent->use_hierarchy) ||
  2915. (memcg->use_hierarchy && !list_empty(&cgrp->children))) {
  2916. cgroup_unlock();
  2917. return -EINVAL;
  2918. }
  2919. spin_lock(&memcg->reclaim_param_lock);
  2920. memcg->swappiness = val;
  2921. spin_unlock(&memcg->reclaim_param_lock);
  2922. cgroup_unlock();
  2923. return 0;
  2924. }
  2925. static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
  2926. {
  2927. struct mem_cgroup_threshold_ary *t;
  2928. u64 usage;
  2929. int i;
  2930. rcu_read_lock();
  2931. if (!swap)
  2932. t = rcu_dereference(memcg->thresholds);
  2933. else
  2934. t = rcu_dereference(memcg->memsw_thresholds);
  2935. if (!t)
  2936. goto unlock;
  2937. usage = mem_cgroup_usage(memcg, swap);
  2938. /*
  2939. * current_threshold points to threshold just below usage.
  2940. * If it's not true, a threshold was crossed after last
  2941. * call of __mem_cgroup_threshold().
  2942. */
  2943. i = atomic_read(&t->current_threshold);
  2944. /*
  2945. * Iterate backward over array of thresholds starting from
  2946. * current_threshold and check if a threshold is crossed.
  2947. * If none of thresholds below usage is crossed, we read
  2948. * only one element of the array here.
  2949. */
  2950. for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
  2951. eventfd_signal(t->entries[i].eventfd, 1);
  2952. /* i = current_threshold + 1 */
  2953. i++;
  2954. /*
  2955. * Iterate forward over array of thresholds starting from
  2956. * current_threshold+1 and check if a threshold is crossed.
  2957. * If none of thresholds above usage is crossed, we read
  2958. * only one element of the array here.
  2959. */
  2960. for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
  2961. eventfd_signal(t->entries[i].eventfd, 1);
  2962. /* Update current_threshold */
  2963. atomic_set(&t->current_threshold, i - 1);
  2964. unlock:
  2965. rcu_read_unlock();
  2966. }
  2967. static void mem_cgroup_threshold(struct mem_cgroup *memcg)
  2968. {
  2969. __mem_cgroup_threshold(memcg, false);
  2970. if (do_swap_account)
  2971. __mem_cgroup_threshold(memcg, true);
  2972. }
  2973. static int compare_thresholds(const void *a, const void *b)
  2974. {
  2975. const struct mem_cgroup_threshold *_a = a;
  2976. const struct mem_cgroup_threshold *_b = b;
  2977. return _a->threshold - _b->threshold;
  2978. }
  2979. static int mem_cgroup_register_event(struct cgroup *cgrp, struct cftype *cft,
  2980. struct eventfd_ctx *eventfd, const char *args)
  2981. {
  2982. struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
  2983. struct mem_cgroup_threshold_ary *thresholds, *thresholds_new;
  2984. int type = MEMFILE_TYPE(cft->private);
  2985. u64 threshold, usage;
  2986. int size;
  2987. int i, ret;
  2988. ret = res_counter_memparse_write_strategy(args, &threshold);
  2989. if (ret)
  2990. return ret;
  2991. mutex_lock(&memcg->thresholds_lock);
  2992. if (type == _MEM)
  2993. thresholds = memcg->thresholds;
  2994. else if (type == _MEMSWAP)
  2995. thresholds = memcg->memsw_thresholds;
  2996. else
  2997. BUG();
  2998. usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
  2999. /* Check if a threshold crossed before adding a new one */
  3000. if (thresholds)
  3001. __mem_cgroup_threshold(memcg, type == _MEMSWAP);
  3002. if (thresholds)
  3003. size = thresholds->size + 1;
  3004. else
  3005. size = 1;
  3006. /* Allocate memory for new array of thresholds */
  3007. thresholds_new = kmalloc(sizeof(*thresholds_new) +
  3008. size * sizeof(struct mem_cgroup_threshold),
  3009. GFP_KERNEL);
  3010. if (!thresholds_new) {
  3011. ret = -ENOMEM;
  3012. goto unlock;
  3013. }
  3014. thresholds_new->size = size;
  3015. /* Copy thresholds (if any) to new array */
  3016. if (thresholds)
  3017. memcpy(thresholds_new->entries, thresholds->entries,
  3018. thresholds->size *
  3019. sizeof(struct mem_cgroup_threshold));
  3020. /* Add new threshold */
  3021. thresholds_new->entries[size - 1].eventfd = eventfd;
  3022. thresholds_new->entries[size - 1].threshold = threshold;
  3023. /* Sort thresholds. Registering of new threshold isn't time-critical */
  3024. sort(thresholds_new->entries, size,
  3025. sizeof(struct mem_cgroup_threshold),
  3026. compare_thresholds, NULL);
  3027. /* Find current threshold */
  3028. atomic_set(&thresholds_new->current_threshold, -1);
  3029. for (i = 0; i < size; i++) {
  3030. if (thresholds_new->entries[i].threshold < usage) {
  3031. /*
  3032. * thresholds_new->current_threshold will not be used
  3033. * until rcu_assign_pointer(), so it's safe to increment
  3034. * it here.
  3035. */
  3036. atomic_inc(&thresholds_new->current_threshold);
  3037. }
  3038. }
  3039. if (type == _MEM)
  3040. rcu_assign_pointer(memcg->thresholds, thresholds_new);
  3041. else
  3042. rcu_assign_pointer(memcg->memsw_thresholds, thresholds_new);
  3043. /* To be sure that nobody uses thresholds before freeing it */
  3044. synchronize_rcu();
  3045. kfree(thresholds);
  3046. unlock:
  3047. mutex_unlock(&memcg->thresholds_lock);
  3048. return ret;
  3049. }
  3050. static int mem_cgroup_unregister_event(struct cgroup *cgrp, struct cftype *cft,
  3051. struct eventfd_ctx *eventfd)
  3052. {
  3053. struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
  3054. struct mem_cgroup_threshold_ary *thresholds, *thresholds_new;
  3055. int type = MEMFILE_TYPE(cft->private);
  3056. u64 usage;
  3057. int size = 0;
  3058. int i, j, ret;
  3059. mutex_lock(&memcg->thresholds_lock);
  3060. if (type == _MEM)
  3061. thresholds = memcg->thresholds;
  3062. else if (type == _MEMSWAP)
  3063. thresholds = memcg->memsw_thresholds;
  3064. else
  3065. BUG();
  3066. /*
  3067. * Something went wrong if we trying to unregister a threshold
  3068. * if we don't have thresholds
  3069. */
  3070. BUG_ON(!thresholds);
  3071. usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
  3072. /* Check if a threshold crossed before removing */
  3073. __mem_cgroup_threshold(memcg, type == _MEMSWAP);
  3074. /* Calculate new number of threshold */
  3075. for (i = 0; i < thresholds->size; i++) {
  3076. if (thresholds->entries[i].eventfd != eventfd)
  3077. size++;
  3078. }
  3079. /* Set thresholds array to NULL if we don't have thresholds */
  3080. if (!size) {
  3081. thresholds_new = NULL;
  3082. goto assign;
  3083. }
  3084. /* Allocate memory for new array of thresholds */
  3085. thresholds_new = kmalloc(sizeof(*thresholds_new) +
  3086. size * sizeof(struct mem_cgroup_threshold),
  3087. GFP_KERNEL);
  3088. if (!thresholds_new) {
  3089. ret = -ENOMEM;
  3090. goto unlock;
  3091. }
  3092. thresholds_new->size = size;
  3093. /* Copy thresholds and find current threshold */
  3094. atomic_set(&thresholds_new->current_threshold, -1);
  3095. for (i = 0, j = 0; i < thresholds->size; i++) {
  3096. if (thresholds->entries[i].eventfd == eventfd)
  3097. continue;
  3098. thresholds_new->entries[j] = thresholds->entries[i];
  3099. if (thresholds_new->entries[j].threshold < usage) {
  3100. /*
  3101. * thresholds_new->current_threshold will not be used
  3102. * until rcu_assign_pointer(), so it's safe to increment
  3103. * it here.
  3104. */
  3105. atomic_inc(&thresholds_new->current_threshold);
  3106. }
  3107. j++;
  3108. }
  3109. assign:
  3110. if (type == _MEM)
  3111. rcu_assign_pointer(memcg->thresholds, thresholds_new);
  3112. else
  3113. rcu_assign_pointer(memcg->memsw_thresholds, thresholds_new);
  3114. /* To be sure that nobody uses thresholds before freeing it */
  3115. synchronize_rcu();
  3116. kfree(thresholds);
  3117. unlock:
  3118. mutex_unlock(&memcg->thresholds_lock);
  3119. return ret;
  3120. }
  3121. static struct cftype mem_cgroup_files[] = {
  3122. {
  3123. .name = "usage_in_bytes",
  3124. .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
  3125. .read_u64 = mem_cgroup_read,
  3126. .register_event = mem_cgroup_register_event,
  3127. .unregister_event = mem_cgroup_unregister_event,
  3128. },
  3129. {
  3130. .name = "max_usage_in_bytes",
  3131. .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
  3132. .trigger = mem_cgroup_reset,
  3133. .read_u64 = mem_cgroup_read,
  3134. },
  3135. {
  3136. .name = "limit_in_bytes",
  3137. .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
  3138. .write_string = mem_cgroup_write,
  3139. .read_u64 = mem_cgroup_read,
  3140. },
  3141. {
  3142. .name = "soft_limit_in_bytes",
  3143. .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
  3144. .write_string = mem_cgroup_write,
  3145. .read_u64 = mem_cgroup_read,
  3146. },
  3147. {
  3148. .name = "failcnt",
  3149. .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
  3150. .trigger = mem_cgroup_reset,
  3151. .read_u64 = mem_cgroup_read,
  3152. },
  3153. {
  3154. .name = "stat",
  3155. .read_map = mem_control_stat_show,
  3156. },
  3157. {
  3158. .name = "force_empty",
  3159. .trigger = mem_cgroup_force_empty_write,
  3160. },
  3161. {
  3162. .name = "use_hierarchy",
  3163. .write_u64 = mem_cgroup_hierarchy_write,
  3164. .read_u64 = mem_cgroup_hierarchy_read,
  3165. },
  3166. {
  3167. .name = "swappiness",
  3168. .read_u64 = mem_cgroup_swappiness_read,
  3169. .write_u64 = mem_cgroup_swappiness_write,
  3170. },
  3171. {
  3172. .name = "move_charge_at_immigrate",
  3173. .read_u64 = mem_cgroup_move_charge_read,
  3174. .write_u64 = mem_cgroup_move_charge_write,
  3175. },
  3176. };
  3177. #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
  3178. static struct cftype memsw_cgroup_files[] = {
  3179. {
  3180. .name = "memsw.usage_in_bytes",
  3181. .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
  3182. .read_u64 = mem_cgroup_read,
  3183. .register_event = mem_cgroup_register_event,
  3184. .unregister_event = mem_cgroup_unregister_event,
  3185. },
  3186. {
  3187. .name = "memsw.max_usage_in_bytes",
  3188. .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
  3189. .trigger = mem_cgroup_reset,
  3190. .read_u64 = mem_cgroup_read,
  3191. },
  3192. {
  3193. .name = "memsw.limit_in_bytes",
  3194. .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
  3195. .write_string = mem_cgroup_write,
  3196. .read_u64 = mem_cgroup_read,
  3197. },
  3198. {
  3199. .name = "memsw.failcnt",
  3200. .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
  3201. .trigger = mem_cgroup_reset,
  3202. .read_u64 = mem_cgroup_read,
  3203. },
  3204. };
  3205. static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
  3206. {
  3207. if (!do_swap_account)
  3208. return 0;
  3209. return cgroup_add_files(cont, ss, memsw_cgroup_files,
  3210. ARRAY_SIZE(memsw_cgroup_files));
  3211. };
  3212. #else
  3213. static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
  3214. {
  3215. return 0;
  3216. }
  3217. #endif
  3218. static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
  3219. {
  3220. struct mem_cgroup_per_node *pn;
  3221. struct mem_cgroup_per_zone *mz;
  3222. enum lru_list l;
  3223. int zone, tmp = node;
  3224. /*
  3225. * This routine is called against possible nodes.
  3226. * But it's BUG to call kmalloc() against offline node.
  3227. *
  3228. * TODO: this routine can waste much memory for nodes which will
  3229. * never be onlined. It's better to use memory hotplug callback
  3230. * function.
  3231. */
  3232. if (!node_state(node, N_NORMAL_MEMORY))
  3233. tmp = -1;
  3234. pn = kmalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
  3235. if (!pn)
  3236. return 1;
  3237. mem->info.nodeinfo[node] = pn;
  3238. memset(pn, 0, sizeof(*pn));
  3239. for (zone = 0; zone < MAX_NR_ZONES; zone++) {
  3240. mz = &pn->zoneinfo[zone];
  3241. for_each_lru(l)
  3242. INIT_LIST_HEAD(&mz->lists[l]);
  3243. mz->usage_in_excess = 0;
  3244. mz->on_tree = false;
  3245. mz->mem = mem;
  3246. }
  3247. return 0;
  3248. }
  3249. static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
  3250. {
  3251. kfree(mem->info.nodeinfo[node]);
  3252. }
  3253. static struct mem_cgroup *mem_cgroup_alloc(void)
  3254. {
  3255. struct mem_cgroup *mem;
  3256. int size = sizeof(struct mem_cgroup);
  3257. /* Can be very big if MAX_NUMNODES is very big */
  3258. if (size < PAGE_SIZE)
  3259. mem = kmalloc(size, GFP_KERNEL);
  3260. else
  3261. mem = vmalloc(size);
  3262. if (!mem)
  3263. return NULL;
  3264. memset(mem, 0, size);
  3265. mem->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
  3266. if (!mem->stat) {
  3267. if (size < PAGE_SIZE)
  3268. kfree(mem);
  3269. else
  3270. vfree(mem);
  3271. mem = NULL;
  3272. }
  3273. return mem;
  3274. }
  3275. /*
  3276. * At destroying mem_cgroup, references from swap_cgroup can remain.
  3277. * (scanning all at force_empty is too costly...)
  3278. *
  3279. * Instead of clearing all references at force_empty, we remember
  3280. * the number of reference from swap_cgroup and free mem_cgroup when
  3281. * it goes down to 0.
  3282. *
  3283. * Removal of cgroup itself succeeds regardless of refs from swap.
  3284. */
  3285. static void __mem_cgroup_free(struct mem_cgroup *mem)
  3286. {
  3287. int node;
  3288. mem_cgroup_remove_from_trees(mem);
  3289. free_css_id(&mem_cgroup_subsys, &mem->css);
  3290. for_each_node_state(node, N_POSSIBLE)
  3291. free_mem_cgroup_per_zone_info(mem, node);
  3292. free_percpu(mem->stat);
  3293. if (sizeof(struct mem_cgroup) < PAGE_SIZE)
  3294. kfree(mem);
  3295. else
  3296. vfree(mem);
  3297. }
  3298. static void mem_cgroup_get(struct mem_cgroup *mem)
  3299. {
  3300. atomic_inc(&mem->refcnt);
  3301. }
  3302. static void __mem_cgroup_put(struct mem_cgroup *mem, int count)
  3303. {
  3304. if (atomic_sub_and_test(count, &mem->refcnt)) {
  3305. struct mem_cgroup *parent = parent_mem_cgroup(mem);
  3306. __mem_cgroup_free(mem);
  3307. if (parent)
  3308. mem_cgroup_put(parent);
  3309. }
  3310. }
  3311. static void mem_cgroup_put(struct mem_cgroup *mem)
  3312. {
  3313. __mem_cgroup_put(mem, 1);
  3314. }
  3315. /*
  3316. * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
  3317. */
  3318. static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem)
  3319. {
  3320. if (!mem->res.parent)
  3321. return NULL;
  3322. return mem_cgroup_from_res_counter(mem->res.parent, res);
  3323. }
  3324. #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
  3325. static void __init enable_swap_cgroup(void)
  3326. {
  3327. if (!mem_cgroup_disabled() && really_do_swap_account)
  3328. do_swap_account = 1;
  3329. }
  3330. #else
  3331. static void __init enable_swap_cgroup(void)
  3332. {
  3333. }
  3334. #endif
  3335. static int mem_cgroup_soft_limit_tree_init(void)
  3336. {
  3337. struct mem_cgroup_tree_per_node *rtpn;
  3338. struct mem_cgroup_tree_per_zone *rtpz;
  3339. int tmp, node, zone;
  3340. for_each_node_state(node, N_POSSIBLE) {
  3341. tmp = node;
  3342. if (!node_state(node, N_NORMAL_MEMORY))
  3343. tmp = -1;
  3344. rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp);
  3345. if (!rtpn)
  3346. return 1;
  3347. soft_limit_tree.rb_tree_per_node[node] = rtpn;
  3348. for (zone = 0; zone < MAX_NR_ZONES; zone++) {
  3349. rtpz = &rtpn->rb_tree_per_zone[zone];
  3350. rtpz->rb_root = RB_ROOT;
  3351. spin_lock_init(&rtpz->lock);
  3352. }
  3353. }
  3354. return 0;
  3355. }
  3356. static struct cgroup_subsys_state * __ref
  3357. mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
  3358. {
  3359. struct mem_cgroup *mem, *parent;
  3360. long error = -ENOMEM;
  3361. int node;
  3362. mem = mem_cgroup_alloc();
  3363. if (!mem)
  3364. return ERR_PTR(error);
  3365. for_each_node_state(node, N_POSSIBLE)
  3366. if (alloc_mem_cgroup_per_zone_info(mem, node))
  3367. goto free_out;
  3368. /* root ? */
  3369. if (cont->parent == NULL) {
  3370. int cpu;
  3371. enable_swap_cgroup();
  3372. parent = NULL;
  3373. root_mem_cgroup = mem;
  3374. if (mem_cgroup_soft_limit_tree_init())
  3375. goto free_out;
  3376. for_each_possible_cpu(cpu) {
  3377. struct memcg_stock_pcp *stock =
  3378. &per_cpu(memcg_stock, cpu);
  3379. INIT_WORK(&stock->work, drain_local_stock);
  3380. }
  3381. hotcpu_notifier(memcg_stock_cpu_callback, 0);
  3382. } else {
  3383. parent = mem_cgroup_from_cont(cont->parent);
  3384. mem->use_hierarchy = parent->use_hierarchy;
  3385. }
  3386. if (parent && parent->use_hierarchy) {
  3387. res_counter_init(&mem->res, &parent->res);
  3388. res_counter_init(&mem->memsw, &parent->memsw);
  3389. /*
  3390. * We increment refcnt of the parent to ensure that we can
  3391. * safely access it on res_counter_charge/uncharge.
  3392. * This refcnt will be decremented when freeing this
  3393. * mem_cgroup(see mem_cgroup_put).
  3394. */
  3395. mem_cgroup_get(parent);
  3396. } else {
  3397. res_counter_init(&mem->res, NULL);
  3398. res_counter_init(&mem->memsw, NULL);
  3399. }
  3400. mem->last_scanned_child = 0;
  3401. spin_lock_init(&mem->reclaim_param_lock);
  3402. if (parent)
  3403. mem->swappiness = get_swappiness(parent);
  3404. atomic_set(&mem->refcnt, 1);
  3405. mem->move_charge_at_immigrate = 0;
  3406. mutex_init(&mem->thresholds_lock);
  3407. return &mem->css;
  3408. free_out:
  3409. __mem_cgroup_free(mem);
  3410. root_mem_cgroup = NULL;
  3411. return ERR_PTR(error);
  3412. }
  3413. static int mem_cgroup_pre_destroy(struct cgroup_subsys *ss,
  3414. struct cgroup *cont)
  3415. {
  3416. struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
  3417. return mem_cgroup_force_empty(mem, false);
  3418. }
  3419. static void mem_cgroup_destroy(struct cgroup_subsys *ss,
  3420. struct cgroup *cont)
  3421. {
  3422. struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
  3423. mem_cgroup_put(mem);
  3424. }
  3425. static int mem_cgroup_populate(struct cgroup_subsys *ss,
  3426. struct cgroup *cont)
  3427. {
  3428. int ret;
  3429. ret = cgroup_add_files(cont, ss, mem_cgroup_files,
  3430. ARRAY_SIZE(mem_cgroup_files));
  3431. if (!ret)
  3432. ret = register_memsw_files(cont, ss);
  3433. return ret;
  3434. }
  3435. #ifdef CONFIG_MMU
  3436. /* Handlers for move charge at task migration. */
  3437. #define PRECHARGE_COUNT_AT_ONCE 256
  3438. static int mem_cgroup_do_precharge(unsigned long count)
  3439. {
  3440. int ret = 0;
  3441. int batch_count = PRECHARGE_COUNT_AT_ONCE;
  3442. struct mem_cgroup *mem = mc.to;
  3443. if (mem_cgroup_is_root(mem)) {
  3444. mc.precharge += count;
  3445. /* we don't need css_get for root */
  3446. return ret;
  3447. }
  3448. /* try to charge at once */
  3449. if (count > 1) {
  3450. struct res_counter *dummy;
  3451. /*
  3452. * "mem" cannot be under rmdir() because we've already checked
  3453. * by cgroup_lock_live_cgroup() that it is not removed and we
  3454. * are still under the same cgroup_mutex. So we can postpone
  3455. * css_get().
  3456. */
  3457. if (res_counter_charge(&mem->res, PAGE_SIZE * count, &dummy))
  3458. goto one_by_one;
  3459. if (do_swap_account && res_counter_charge(&mem->memsw,
  3460. PAGE_SIZE * count, &dummy)) {
  3461. res_counter_uncharge(&mem->res, PAGE_SIZE * count);
  3462. goto one_by_one;
  3463. }
  3464. mc.precharge += count;
  3465. VM_BUG_ON(test_bit(CSS_ROOT, &mem->css.flags));
  3466. WARN_ON_ONCE(count > INT_MAX);
  3467. __css_get(&mem->css, (int)count);
  3468. return ret;
  3469. }
  3470. one_by_one:
  3471. /* fall back to one by one charge */
  3472. while (count--) {
  3473. if (signal_pending(current)) {
  3474. ret = -EINTR;
  3475. break;
  3476. }
  3477. if (!batch_count--) {
  3478. batch_count = PRECHARGE_COUNT_AT_ONCE;
  3479. cond_resched();
  3480. }
  3481. ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem, false);
  3482. if (ret || !mem)
  3483. /* mem_cgroup_clear_mc() will do uncharge later */
  3484. return -ENOMEM;
  3485. mc.precharge++;
  3486. }
  3487. return ret;
  3488. }
  3489. /**
  3490. * is_target_pte_for_mc - check a pte whether it is valid for move charge
  3491. * @vma: the vma the pte to be checked belongs
  3492. * @addr: the address corresponding to the pte to be checked
  3493. * @ptent: the pte to be checked
  3494. * @target: the pointer the target page or swap ent will be stored(can be NULL)
  3495. *
  3496. * Returns
  3497. * 0(MC_TARGET_NONE): if the pte is not a target for move charge.
  3498. * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
  3499. * move charge. if @target is not NULL, the page is stored in target->page
  3500. * with extra refcnt got(Callers should handle it).
  3501. * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
  3502. * target for charge migration. if @target is not NULL, the entry is stored
  3503. * in target->ent.
  3504. *
  3505. * Called with pte lock held.
  3506. */
  3507. union mc_target {
  3508. struct page *page;
  3509. swp_entry_t ent;
  3510. };
  3511. enum mc_target_type {
  3512. MC_TARGET_NONE, /* not used */
  3513. MC_TARGET_PAGE,
  3514. MC_TARGET_SWAP,
  3515. };
  3516. static int is_target_pte_for_mc(struct vm_area_struct *vma,
  3517. unsigned long addr, pte_t ptent, union mc_target *target)
  3518. {
  3519. struct page *page = NULL;
  3520. struct page_cgroup *pc;
  3521. int ret = 0;
  3522. swp_entry_t ent = { .val = 0 };
  3523. int usage_count = 0;
  3524. bool move_anon = test_bit(MOVE_CHARGE_TYPE_ANON,
  3525. &mc.to->move_charge_at_immigrate);
  3526. if (!pte_present(ptent)) {
  3527. /* TODO: handle swap of shmes/tmpfs */
  3528. if (pte_none(ptent) || pte_file(ptent))
  3529. return 0;
  3530. else if (is_swap_pte(ptent)) {
  3531. ent = pte_to_swp_entry(ptent);
  3532. if (!move_anon || non_swap_entry(ent))
  3533. return 0;
  3534. usage_count = mem_cgroup_count_swap_user(ent, &page);
  3535. }
  3536. } else {
  3537. page = vm_normal_page(vma, addr, ptent);
  3538. if (!page || !page_mapped(page))
  3539. return 0;
  3540. /*
  3541. * TODO: We don't move charges of file(including shmem/tmpfs)
  3542. * pages for now.
  3543. */
  3544. if (!move_anon || !PageAnon(page))
  3545. return 0;
  3546. if (!get_page_unless_zero(page))
  3547. return 0;
  3548. usage_count = page_mapcount(page);
  3549. }
  3550. if (usage_count > 1) {
  3551. /*
  3552. * TODO: We don't move charges of shared(used by multiple
  3553. * processes) pages for now.
  3554. */
  3555. if (page)
  3556. put_page(page);
  3557. return 0;
  3558. }
  3559. if (page) {
  3560. pc = lookup_page_cgroup(page);
  3561. /*
  3562. * Do only loose check w/o page_cgroup lock.
  3563. * mem_cgroup_move_account() checks the pc is valid or not under
  3564. * the lock.
  3565. */
  3566. if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
  3567. ret = MC_TARGET_PAGE;
  3568. if (target)
  3569. target->page = page;
  3570. }
  3571. if (!ret || !target)
  3572. put_page(page);
  3573. }
  3574. /* throught */
  3575. if (ent.val && do_swap_account && !ret) {
  3576. unsigned short id;
  3577. rcu_read_lock();
  3578. id = css_id(&mc.from->css);
  3579. rcu_read_unlock();
  3580. if (id == lookup_swap_cgroup(ent)) {
  3581. ret = MC_TARGET_SWAP;
  3582. if (target)
  3583. target->ent = ent;
  3584. }
  3585. }
  3586. return ret;
  3587. }
  3588. static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
  3589. unsigned long addr, unsigned long end,
  3590. struct mm_walk *walk)
  3591. {
  3592. struct vm_area_struct *vma = walk->private;
  3593. pte_t *pte;
  3594. spinlock_t *ptl;
  3595. pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
  3596. for (; addr != end; pte++, addr += PAGE_SIZE)
  3597. if (is_target_pte_for_mc(vma, addr, *pte, NULL))
  3598. mc.precharge++; /* increment precharge temporarily */
  3599. pte_unmap_unlock(pte - 1, ptl);
  3600. cond_resched();
  3601. return 0;
  3602. }
  3603. static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
  3604. {
  3605. unsigned long precharge;
  3606. struct vm_area_struct *vma;
  3607. down_read(&mm->mmap_sem);
  3608. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  3609. struct mm_walk mem_cgroup_count_precharge_walk = {
  3610. .pmd_entry = mem_cgroup_count_precharge_pte_range,
  3611. .mm = mm,
  3612. .private = vma,
  3613. };
  3614. if (is_vm_hugetlb_page(vma))
  3615. continue;
  3616. /* TODO: We don't move charges of shmem/tmpfs pages for now. */
  3617. if (vma->vm_flags & VM_SHARED)
  3618. continue;
  3619. walk_page_range(vma->vm_start, vma->vm_end,
  3620. &mem_cgroup_count_precharge_walk);
  3621. }
  3622. up_read(&mm->mmap_sem);
  3623. precharge = mc.precharge;
  3624. mc.precharge = 0;
  3625. return precharge;
  3626. }
  3627. static int mem_cgroup_precharge_mc(struct mm_struct *mm)
  3628. {
  3629. return mem_cgroup_do_precharge(mem_cgroup_count_precharge(mm));
  3630. }
  3631. static void mem_cgroup_clear_mc(void)
  3632. {
  3633. /* we must uncharge all the leftover precharges from mc.to */
  3634. if (mc.precharge) {
  3635. __mem_cgroup_cancel_charge(mc.to, mc.precharge);
  3636. mc.precharge = 0;
  3637. }
  3638. /*
  3639. * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
  3640. * we must uncharge here.
  3641. */
  3642. if (mc.moved_charge) {
  3643. __mem_cgroup_cancel_charge(mc.from, mc.moved_charge);
  3644. mc.moved_charge = 0;
  3645. }
  3646. /* we must fixup refcnts and charges */
  3647. if (mc.moved_swap) {
  3648. WARN_ON_ONCE(mc.moved_swap > INT_MAX);
  3649. /* uncharge swap account from the old cgroup */
  3650. if (!mem_cgroup_is_root(mc.from))
  3651. res_counter_uncharge(&mc.from->memsw,
  3652. PAGE_SIZE * mc.moved_swap);
  3653. __mem_cgroup_put(mc.from, mc.moved_swap);
  3654. if (!mem_cgroup_is_root(mc.to)) {
  3655. /*
  3656. * we charged both to->res and to->memsw, so we should
  3657. * uncharge to->res.
  3658. */
  3659. res_counter_uncharge(&mc.to->res,
  3660. PAGE_SIZE * mc.moved_swap);
  3661. VM_BUG_ON(test_bit(CSS_ROOT, &mc.to->css.flags));
  3662. __css_put(&mc.to->css, mc.moved_swap);
  3663. }
  3664. /* we've already done mem_cgroup_get(mc.to) */
  3665. mc.moved_swap = 0;
  3666. }
  3667. mc.from = NULL;
  3668. mc.to = NULL;
  3669. mc.moving_task = NULL;
  3670. wake_up_all(&mc.waitq);
  3671. }
  3672. static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
  3673. struct cgroup *cgroup,
  3674. struct task_struct *p,
  3675. bool threadgroup)
  3676. {
  3677. int ret = 0;
  3678. struct mem_cgroup *mem = mem_cgroup_from_cont(cgroup);
  3679. if (mem->move_charge_at_immigrate) {
  3680. struct mm_struct *mm;
  3681. struct mem_cgroup *from = mem_cgroup_from_task(p);
  3682. VM_BUG_ON(from == mem);
  3683. mm = get_task_mm(p);
  3684. if (!mm)
  3685. return 0;
  3686. /* We move charges only when we move a owner of the mm */
  3687. if (mm->owner == p) {
  3688. VM_BUG_ON(mc.from);
  3689. VM_BUG_ON(mc.to);
  3690. VM_BUG_ON(mc.precharge);
  3691. VM_BUG_ON(mc.moved_charge);
  3692. VM_BUG_ON(mc.moved_swap);
  3693. VM_BUG_ON(mc.moving_task);
  3694. mc.from = from;
  3695. mc.to = mem;
  3696. mc.precharge = 0;
  3697. mc.moved_charge = 0;
  3698. mc.moved_swap = 0;
  3699. mc.moving_task = current;
  3700. ret = mem_cgroup_precharge_mc(mm);
  3701. if (ret)
  3702. mem_cgroup_clear_mc();
  3703. }
  3704. mmput(mm);
  3705. }
  3706. return ret;
  3707. }
  3708. static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
  3709. struct cgroup *cgroup,
  3710. struct task_struct *p,
  3711. bool threadgroup)
  3712. {
  3713. mem_cgroup_clear_mc();
  3714. }
  3715. static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
  3716. unsigned long addr, unsigned long end,
  3717. struct mm_walk *walk)
  3718. {
  3719. int ret = 0;
  3720. struct vm_area_struct *vma = walk->private;
  3721. pte_t *pte;
  3722. spinlock_t *ptl;
  3723. retry:
  3724. pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
  3725. for (; addr != end; addr += PAGE_SIZE) {
  3726. pte_t ptent = *(pte++);
  3727. union mc_target target;
  3728. int type;
  3729. struct page *page;
  3730. struct page_cgroup *pc;
  3731. swp_entry_t ent;
  3732. if (!mc.precharge)
  3733. break;
  3734. type = is_target_pte_for_mc(vma, addr, ptent, &target);
  3735. switch (type) {
  3736. case MC_TARGET_PAGE:
  3737. page = target.page;
  3738. if (isolate_lru_page(page))
  3739. goto put;
  3740. pc = lookup_page_cgroup(page);
  3741. if (!mem_cgroup_move_account(pc,
  3742. mc.from, mc.to, false)) {
  3743. mc.precharge--;
  3744. /* we uncharge from mc.from later. */
  3745. mc.moved_charge++;
  3746. }
  3747. putback_lru_page(page);
  3748. put: /* is_target_pte_for_mc() gets the page */
  3749. put_page(page);
  3750. break;
  3751. case MC_TARGET_SWAP:
  3752. ent = target.ent;
  3753. if (!mem_cgroup_move_swap_account(ent,
  3754. mc.from, mc.to, false)) {
  3755. mc.precharge--;
  3756. /* we fixup refcnts and charges later. */
  3757. mc.moved_swap++;
  3758. }
  3759. break;
  3760. default:
  3761. break;
  3762. }
  3763. }
  3764. pte_unmap_unlock(pte - 1, ptl);
  3765. cond_resched();
  3766. if (addr != end) {
  3767. /*
  3768. * We have consumed all precharges we got in can_attach().
  3769. * We try charge one by one, but don't do any additional
  3770. * charges to mc.to if we have failed in charge once in attach()
  3771. * phase.
  3772. */
  3773. ret = mem_cgroup_do_precharge(1);
  3774. if (!ret)
  3775. goto retry;
  3776. }
  3777. return ret;
  3778. }
  3779. static void mem_cgroup_move_charge(struct mm_struct *mm)
  3780. {
  3781. struct vm_area_struct *vma;
  3782. lru_add_drain_all();
  3783. down_read(&mm->mmap_sem);
  3784. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  3785. int ret;
  3786. struct mm_walk mem_cgroup_move_charge_walk = {
  3787. .pmd_entry = mem_cgroup_move_charge_pte_range,
  3788. .mm = mm,
  3789. .private = vma,
  3790. };
  3791. if (is_vm_hugetlb_page(vma))
  3792. continue;
  3793. /* TODO: We don't move charges of shmem/tmpfs pages for now. */
  3794. if (vma->vm_flags & VM_SHARED)
  3795. continue;
  3796. ret = walk_page_range(vma->vm_start, vma->vm_end,
  3797. &mem_cgroup_move_charge_walk);
  3798. if (ret)
  3799. /*
  3800. * means we have consumed all precharges and failed in
  3801. * doing additional charge. Just abandon here.
  3802. */
  3803. break;
  3804. }
  3805. up_read(&mm->mmap_sem);
  3806. }
  3807. static void mem_cgroup_move_task(struct cgroup_subsys *ss,
  3808. struct cgroup *cont,
  3809. struct cgroup *old_cont,
  3810. struct task_struct *p,
  3811. bool threadgroup)
  3812. {
  3813. struct mm_struct *mm;
  3814. if (!mc.to)
  3815. /* no need to move charge */
  3816. return;
  3817. mm = get_task_mm(p);
  3818. if (mm) {
  3819. mem_cgroup_move_charge(mm);
  3820. mmput(mm);
  3821. }
  3822. mem_cgroup_clear_mc();
  3823. }
  3824. #else /* !CONFIG_MMU */
  3825. static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
  3826. struct cgroup *cgroup,
  3827. struct task_struct *p,
  3828. bool threadgroup)
  3829. {
  3830. return 0;
  3831. }
  3832. static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
  3833. struct cgroup *cgroup,
  3834. struct task_struct *p,
  3835. bool threadgroup)
  3836. {
  3837. }
  3838. static void mem_cgroup_move_task(struct cgroup_subsys *ss,
  3839. struct cgroup *cont,
  3840. struct cgroup *old_cont,
  3841. struct task_struct *p,
  3842. bool threadgroup)
  3843. {
  3844. }
  3845. #endif
  3846. struct cgroup_subsys mem_cgroup_subsys = {
  3847. .name = "memory",
  3848. .subsys_id = mem_cgroup_subsys_id,
  3849. .create = mem_cgroup_create,
  3850. .pre_destroy = mem_cgroup_pre_destroy,
  3851. .destroy = mem_cgroup_destroy,
  3852. .populate = mem_cgroup_populate,
  3853. .can_attach = mem_cgroup_can_attach,
  3854. .cancel_attach = mem_cgroup_cancel_attach,
  3855. .attach = mem_cgroup_move_task,
  3856. .early_init = 0,
  3857. .use_id = 1,
  3858. };
  3859. #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
  3860. static int __init disable_swap_account(char *s)
  3861. {
  3862. really_do_swap_account = 0;
  3863. return 1;
  3864. }
  3865. __setup("noswapaccount", disable_swap_account);
  3866. #endif