memcontrol.c 27 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109
  1. /* memcontrol.c - Memory Controller
  2. *
  3. * Copyright IBM Corporation, 2007
  4. * Author Balbir Singh <balbir@linux.vnet.ibm.com>
  5. *
  6. * Copyright 2007 OpenVZ SWsoft Inc
  7. * Author: Pavel Emelianov <xemul@openvz.org>
  8. *
  9. * This program is free software; you can redistribute it and/or modify
  10. * it under the terms of the GNU General Public License as published by
  11. * the Free Software Foundation; either version 2 of the License, or
  12. * (at your option) any later version.
  13. *
  14. * This program is distributed in the hope that it will be useful,
  15. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17. * GNU General Public License for more details.
  18. */
  19. #include <linux/res_counter.h>
  20. #include <linux/memcontrol.h>
  21. #include <linux/cgroup.h>
  22. #include <linux/mm.h>
  23. #include <linux/smp.h>
  24. #include <linux/page-flags.h>
  25. #include <linux/backing-dev.h>
  26. #include <linux/bit_spinlock.h>
  27. #include <linux/rcupdate.h>
  28. #include <linux/slab.h>
  29. #include <linux/swap.h>
  30. #include <linux/spinlock.h>
  31. #include <linux/fs.h>
  32. #include <linux/seq_file.h>
  33. #include <linux/vmalloc.h>
  34. #include <asm/uaccess.h>
  35. struct cgroup_subsys mem_cgroup_subsys;
  36. static const int MEM_CGROUP_RECLAIM_RETRIES = 5;
  37. static struct kmem_cache *page_cgroup_cache;
  38. /*
  39. * Statistics for memory cgroup.
  40. */
  41. enum mem_cgroup_stat_index {
  42. /*
  43. * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
  44. */
  45. MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */
  46. MEM_CGROUP_STAT_RSS, /* # of pages charged as rss */
  47. MEM_CGROUP_STAT_NSTATS,
  48. };
  49. struct mem_cgroup_stat_cpu {
  50. s64 count[MEM_CGROUP_STAT_NSTATS];
  51. } ____cacheline_aligned_in_smp;
  52. struct mem_cgroup_stat {
  53. struct mem_cgroup_stat_cpu cpustat[NR_CPUS];
  54. };
  55. /*
  56. * For accounting under irq disable, no need for increment preempt count.
  57. */
  58. static void __mem_cgroup_stat_add_safe(struct mem_cgroup_stat *stat,
  59. enum mem_cgroup_stat_index idx, int val)
  60. {
  61. int cpu = smp_processor_id();
  62. stat->cpustat[cpu].count[idx] += val;
  63. }
  64. static s64 mem_cgroup_read_stat(struct mem_cgroup_stat *stat,
  65. enum mem_cgroup_stat_index idx)
  66. {
  67. int cpu;
  68. s64 ret = 0;
  69. for_each_possible_cpu(cpu)
  70. ret += stat->cpustat[cpu].count[idx];
  71. return ret;
  72. }
  73. /*
  74. * per-zone information in memory controller.
  75. */
  76. enum mem_cgroup_zstat_index {
  77. MEM_CGROUP_ZSTAT_ACTIVE,
  78. MEM_CGROUP_ZSTAT_INACTIVE,
  79. NR_MEM_CGROUP_ZSTAT,
  80. };
  81. struct mem_cgroup_per_zone {
  82. /*
  83. * spin_lock to protect the per cgroup LRU
  84. */
  85. spinlock_t lru_lock;
  86. struct list_head active_list;
  87. struct list_head inactive_list;
  88. unsigned long count[NR_MEM_CGROUP_ZSTAT];
  89. };
  90. /* Macro for accessing counter */
  91. #define MEM_CGROUP_ZSTAT(mz, idx) ((mz)->count[(idx)])
  92. struct mem_cgroup_per_node {
  93. struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
  94. };
  95. struct mem_cgroup_lru_info {
  96. struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES];
  97. };
  98. /*
  99. * The memory controller data structure. The memory controller controls both
  100. * page cache and RSS per cgroup. We would eventually like to provide
  101. * statistics based on the statistics developed by Rik Van Riel for clock-pro,
  102. * to help the administrator determine what knobs to tune.
  103. *
  104. * TODO: Add a water mark for the memory controller. Reclaim will begin when
  105. * we hit the water mark. May be even add a low water mark, such that
  106. * no reclaim occurs from a cgroup at it's low water mark, this is
  107. * a feature that will be implemented much later in the future.
  108. */
  109. struct mem_cgroup {
  110. struct cgroup_subsys_state css;
  111. /*
  112. * the counter to account for memory usage
  113. */
  114. struct res_counter res;
  115. /*
  116. * Per cgroup active and inactive list, similar to the
  117. * per zone LRU lists.
  118. */
  119. struct mem_cgroup_lru_info info;
  120. int prev_priority; /* for recording reclaim priority */
  121. /*
  122. * statistics.
  123. */
  124. struct mem_cgroup_stat stat;
  125. };
  126. static struct mem_cgroup init_mem_cgroup;
  127. /*
  128. * We use the lower bit of the page->page_cgroup pointer as a bit spin
  129. * lock. We need to ensure that page->page_cgroup is at least two
  130. * byte aligned (based on comments from Nick Piggin). But since
  131. * bit_spin_lock doesn't actually set that lock bit in a non-debug
  132. * uniprocessor kernel, we should avoid setting it here too.
  133. */
  134. #define PAGE_CGROUP_LOCK_BIT 0x0
  135. #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
  136. #define PAGE_CGROUP_LOCK (1 << PAGE_CGROUP_LOCK_BIT)
  137. #else
  138. #define PAGE_CGROUP_LOCK 0x0
  139. #endif
  140. /*
  141. * A page_cgroup page is associated with every page descriptor. The
  142. * page_cgroup helps us identify information about the cgroup
  143. */
  144. struct page_cgroup {
  145. struct list_head lru; /* per cgroup LRU list */
  146. struct page *page;
  147. struct mem_cgroup *mem_cgroup;
  148. int ref_cnt; /* cached, mapped, migrating */
  149. int flags;
  150. };
  151. #define PAGE_CGROUP_FLAG_CACHE (0x1) /* charged as cache */
  152. #define PAGE_CGROUP_FLAG_ACTIVE (0x2) /* page is active in this cgroup */
  153. static int page_cgroup_nid(struct page_cgroup *pc)
  154. {
  155. return page_to_nid(pc->page);
  156. }
  157. static enum zone_type page_cgroup_zid(struct page_cgroup *pc)
  158. {
  159. return page_zonenum(pc->page);
  160. }
  161. enum charge_type {
  162. MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
  163. MEM_CGROUP_CHARGE_TYPE_MAPPED,
  164. };
  165. /*
  166. * Always modified under lru lock. Then, not necessary to preempt_disable()
  167. */
  168. static void mem_cgroup_charge_statistics(struct mem_cgroup *mem, int flags,
  169. bool charge)
  170. {
  171. int val = (charge)? 1 : -1;
  172. struct mem_cgroup_stat *stat = &mem->stat;
  173. VM_BUG_ON(!irqs_disabled());
  174. if (flags & PAGE_CGROUP_FLAG_CACHE)
  175. __mem_cgroup_stat_add_safe(stat, MEM_CGROUP_STAT_CACHE, val);
  176. else
  177. __mem_cgroup_stat_add_safe(stat, MEM_CGROUP_STAT_RSS, val);
  178. }
  179. static struct mem_cgroup_per_zone *
  180. mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid)
  181. {
  182. return &mem->info.nodeinfo[nid]->zoneinfo[zid];
  183. }
  184. static struct mem_cgroup_per_zone *
  185. page_cgroup_zoneinfo(struct page_cgroup *pc)
  186. {
  187. struct mem_cgroup *mem = pc->mem_cgroup;
  188. int nid = page_cgroup_nid(pc);
  189. int zid = page_cgroup_zid(pc);
  190. return mem_cgroup_zoneinfo(mem, nid, zid);
  191. }
  192. static unsigned long mem_cgroup_get_all_zonestat(struct mem_cgroup *mem,
  193. enum mem_cgroup_zstat_index idx)
  194. {
  195. int nid, zid;
  196. struct mem_cgroup_per_zone *mz;
  197. u64 total = 0;
  198. for_each_online_node(nid)
  199. for (zid = 0; zid < MAX_NR_ZONES; zid++) {
  200. mz = mem_cgroup_zoneinfo(mem, nid, zid);
  201. total += MEM_CGROUP_ZSTAT(mz, idx);
  202. }
  203. return total;
  204. }
  205. static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
  206. {
  207. return container_of(cgroup_subsys_state(cont,
  208. mem_cgroup_subsys_id), struct mem_cgroup,
  209. css);
  210. }
  211. struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
  212. {
  213. return container_of(task_subsys_state(p, mem_cgroup_subsys_id),
  214. struct mem_cgroup, css);
  215. }
  216. static inline int page_cgroup_locked(struct page *page)
  217. {
  218. return bit_spin_is_locked(PAGE_CGROUP_LOCK_BIT, &page->page_cgroup);
  219. }
  220. static void page_assign_page_cgroup(struct page *page, struct page_cgroup *pc)
  221. {
  222. VM_BUG_ON(!page_cgroup_locked(page));
  223. page->page_cgroup = ((unsigned long)pc | PAGE_CGROUP_LOCK);
  224. }
  225. struct page_cgroup *page_get_page_cgroup(struct page *page)
  226. {
  227. return (struct page_cgroup *) (page->page_cgroup & ~PAGE_CGROUP_LOCK);
  228. }
  229. static void lock_page_cgroup(struct page *page)
  230. {
  231. bit_spin_lock(PAGE_CGROUP_LOCK_BIT, &page->page_cgroup);
  232. }
  233. static int try_lock_page_cgroup(struct page *page)
  234. {
  235. return bit_spin_trylock(PAGE_CGROUP_LOCK_BIT, &page->page_cgroup);
  236. }
  237. static void unlock_page_cgroup(struct page *page)
  238. {
  239. bit_spin_unlock(PAGE_CGROUP_LOCK_BIT, &page->page_cgroup);
  240. }
  241. static void __mem_cgroup_remove_list(struct mem_cgroup_per_zone *mz,
  242. struct page_cgroup *pc)
  243. {
  244. int from = pc->flags & PAGE_CGROUP_FLAG_ACTIVE;
  245. if (from)
  246. MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE) -= 1;
  247. else
  248. MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE) -= 1;
  249. mem_cgroup_charge_statistics(pc->mem_cgroup, pc->flags, false);
  250. list_del_init(&pc->lru);
  251. }
  252. static void __mem_cgroup_add_list(struct mem_cgroup_per_zone *mz,
  253. struct page_cgroup *pc)
  254. {
  255. int to = pc->flags & PAGE_CGROUP_FLAG_ACTIVE;
  256. if (!to) {
  257. MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE) += 1;
  258. list_add(&pc->lru, &mz->inactive_list);
  259. } else {
  260. MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE) += 1;
  261. list_add(&pc->lru, &mz->active_list);
  262. }
  263. mem_cgroup_charge_statistics(pc->mem_cgroup, pc->flags, true);
  264. }
  265. static void __mem_cgroup_move_lists(struct page_cgroup *pc, bool active)
  266. {
  267. int from = pc->flags & PAGE_CGROUP_FLAG_ACTIVE;
  268. struct mem_cgroup_per_zone *mz = page_cgroup_zoneinfo(pc);
  269. if (from)
  270. MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE) -= 1;
  271. else
  272. MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE) -= 1;
  273. if (active) {
  274. MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE) += 1;
  275. pc->flags |= PAGE_CGROUP_FLAG_ACTIVE;
  276. list_move(&pc->lru, &mz->active_list);
  277. } else {
  278. MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE) += 1;
  279. pc->flags &= ~PAGE_CGROUP_FLAG_ACTIVE;
  280. list_move(&pc->lru, &mz->inactive_list);
  281. }
  282. }
  283. int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
  284. {
  285. int ret;
  286. task_lock(task);
  287. ret = task->mm && mm_match_cgroup(task->mm, mem);
  288. task_unlock(task);
  289. return ret;
  290. }
  291. /*
  292. * This routine assumes that the appropriate zone's lru lock is already held
  293. */
  294. void mem_cgroup_move_lists(struct page *page, bool active)
  295. {
  296. struct page_cgroup *pc;
  297. struct mem_cgroup_per_zone *mz;
  298. unsigned long flags;
  299. /*
  300. * We cannot lock_page_cgroup while holding zone's lru_lock,
  301. * because other holders of lock_page_cgroup can be interrupted
  302. * with an attempt to rotate_reclaimable_page. But we cannot
  303. * safely get to page_cgroup without it, so just try_lock it:
  304. * mem_cgroup_isolate_pages allows for page left on wrong list.
  305. */
  306. if (!try_lock_page_cgroup(page))
  307. return;
  308. pc = page_get_page_cgroup(page);
  309. if (pc) {
  310. mz = page_cgroup_zoneinfo(pc);
  311. spin_lock_irqsave(&mz->lru_lock, flags);
  312. __mem_cgroup_move_lists(pc, active);
  313. spin_unlock_irqrestore(&mz->lru_lock, flags);
  314. }
  315. unlock_page_cgroup(page);
  316. }
  317. /*
  318. * Calculate mapped_ratio under memory controller. This will be used in
  319. * vmscan.c for deteremining we have to reclaim mapped pages.
  320. */
  321. int mem_cgroup_calc_mapped_ratio(struct mem_cgroup *mem)
  322. {
  323. long total, rss;
  324. /*
  325. * usage is recorded in bytes. But, here, we assume the number of
  326. * physical pages can be represented by "long" on any arch.
  327. */
  328. total = (long) (mem->res.usage >> PAGE_SHIFT) + 1L;
  329. rss = (long)mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_RSS);
  330. return (int)((rss * 100L) / total);
  331. }
  332. /*
  333. * This function is called from vmscan.c. In page reclaiming loop. balance
  334. * between active and inactive list is calculated. For memory controller
  335. * page reclaiming, we should use using mem_cgroup's imbalance rather than
  336. * zone's global lru imbalance.
  337. */
  338. long mem_cgroup_reclaim_imbalance(struct mem_cgroup *mem)
  339. {
  340. unsigned long active, inactive;
  341. /* active and inactive are the number of pages. 'long' is ok.*/
  342. active = mem_cgroup_get_all_zonestat(mem, MEM_CGROUP_ZSTAT_ACTIVE);
  343. inactive = mem_cgroup_get_all_zonestat(mem, MEM_CGROUP_ZSTAT_INACTIVE);
  344. return (long) (active / (inactive + 1));
  345. }
  346. /*
  347. * prev_priority control...this will be used in memory reclaim path.
  348. */
  349. int mem_cgroup_get_reclaim_priority(struct mem_cgroup *mem)
  350. {
  351. return mem->prev_priority;
  352. }
  353. void mem_cgroup_note_reclaim_priority(struct mem_cgroup *mem, int priority)
  354. {
  355. if (priority < mem->prev_priority)
  356. mem->prev_priority = priority;
  357. }
  358. void mem_cgroup_record_reclaim_priority(struct mem_cgroup *mem, int priority)
  359. {
  360. mem->prev_priority = priority;
  361. }
  362. /*
  363. * Calculate # of pages to be scanned in this priority/zone.
  364. * See also vmscan.c
  365. *
  366. * priority starts from "DEF_PRIORITY" and decremented in each loop.
  367. * (see include/linux/mmzone.h)
  368. */
  369. long mem_cgroup_calc_reclaim_active(struct mem_cgroup *mem,
  370. struct zone *zone, int priority)
  371. {
  372. long nr_active;
  373. int nid = zone->zone_pgdat->node_id;
  374. int zid = zone_idx(zone);
  375. struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(mem, nid, zid);
  376. nr_active = MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE);
  377. return (nr_active >> priority);
  378. }
  379. long mem_cgroup_calc_reclaim_inactive(struct mem_cgroup *mem,
  380. struct zone *zone, int priority)
  381. {
  382. long nr_inactive;
  383. int nid = zone->zone_pgdat->node_id;
  384. int zid = zone_idx(zone);
  385. struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(mem, nid, zid);
  386. nr_inactive = MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE);
  387. return (nr_inactive >> priority);
  388. }
  389. unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
  390. struct list_head *dst,
  391. unsigned long *scanned, int order,
  392. int mode, struct zone *z,
  393. struct mem_cgroup *mem_cont,
  394. int active)
  395. {
  396. unsigned long nr_taken = 0;
  397. struct page *page;
  398. unsigned long scan;
  399. LIST_HEAD(pc_list);
  400. struct list_head *src;
  401. struct page_cgroup *pc, *tmp;
  402. int nid = z->zone_pgdat->node_id;
  403. int zid = zone_idx(z);
  404. struct mem_cgroup_per_zone *mz;
  405. BUG_ON(!mem_cont);
  406. mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
  407. if (active)
  408. src = &mz->active_list;
  409. else
  410. src = &mz->inactive_list;
  411. spin_lock(&mz->lru_lock);
  412. scan = 0;
  413. list_for_each_entry_safe_reverse(pc, tmp, src, lru) {
  414. if (scan >= nr_to_scan)
  415. break;
  416. page = pc->page;
  417. if (unlikely(!PageLRU(page)))
  418. continue;
  419. if (PageActive(page) && !active) {
  420. __mem_cgroup_move_lists(pc, true);
  421. continue;
  422. }
  423. if (!PageActive(page) && active) {
  424. __mem_cgroup_move_lists(pc, false);
  425. continue;
  426. }
  427. scan++;
  428. list_move(&pc->lru, &pc_list);
  429. if (__isolate_lru_page(page, mode) == 0) {
  430. list_move(&page->lru, dst);
  431. nr_taken++;
  432. }
  433. }
  434. list_splice(&pc_list, src);
  435. spin_unlock(&mz->lru_lock);
  436. *scanned = scan;
  437. return nr_taken;
  438. }
  439. /*
  440. * Charge the memory controller for page usage.
  441. * Return
  442. * 0 if the charge was successful
  443. * < 0 if the cgroup is over its limit
  444. */
  445. static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
  446. gfp_t gfp_mask, enum charge_type ctype)
  447. {
  448. struct mem_cgroup *mem;
  449. struct page_cgroup *pc;
  450. unsigned long flags;
  451. unsigned long nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
  452. struct mem_cgroup_per_zone *mz;
  453. if (mem_cgroup_subsys.disabled)
  454. return 0;
  455. /*
  456. * Should page_cgroup's go to their own slab?
  457. * One could optimize the performance of the charging routine
  458. * by saving a bit in the page_flags and using it as a lock
  459. * to see if the cgroup page already has a page_cgroup associated
  460. * with it
  461. */
  462. retry:
  463. lock_page_cgroup(page);
  464. pc = page_get_page_cgroup(page);
  465. /*
  466. * The page_cgroup exists and
  467. * the page has already been accounted.
  468. */
  469. if (pc) {
  470. VM_BUG_ON(pc->page != page);
  471. VM_BUG_ON(pc->ref_cnt <= 0);
  472. pc->ref_cnt++;
  473. unlock_page_cgroup(page);
  474. goto done;
  475. }
  476. unlock_page_cgroup(page);
  477. pc = kmem_cache_zalloc(page_cgroup_cache, gfp_mask);
  478. if (pc == NULL)
  479. goto err;
  480. /*
  481. * We always charge the cgroup the mm_struct belongs to.
  482. * The mm_struct's mem_cgroup changes on task migration if the
  483. * thread group leader migrates. It's possible that mm is not
  484. * set, if so charge the init_mm (happens for pagecache usage).
  485. */
  486. if (!mm)
  487. mm = &init_mm;
  488. rcu_read_lock();
  489. mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
  490. /*
  491. * For every charge from the cgroup, increment reference count
  492. */
  493. css_get(&mem->css);
  494. rcu_read_unlock();
  495. while (res_counter_charge(&mem->res, PAGE_SIZE)) {
  496. if (!(gfp_mask & __GFP_WAIT))
  497. goto out;
  498. if (try_to_free_mem_cgroup_pages(mem, gfp_mask))
  499. continue;
  500. /*
  501. * try_to_free_mem_cgroup_pages() might not give us a full
  502. * picture of reclaim. Some pages are reclaimed and might be
  503. * moved to swap cache or just unmapped from the cgroup.
  504. * Check the limit again to see if the reclaim reduced the
  505. * current usage of the cgroup before giving up
  506. */
  507. if (res_counter_check_under_limit(&mem->res))
  508. continue;
  509. if (!nr_retries--) {
  510. mem_cgroup_out_of_memory(mem, gfp_mask);
  511. goto out;
  512. }
  513. }
  514. pc->ref_cnt = 1;
  515. pc->mem_cgroup = mem;
  516. pc->page = page;
  517. pc->flags = PAGE_CGROUP_FLAG_ACTIVE;
  518. if (ctype == MEM_CGROUP_CHARGE_TYPE_CACHE)
  519. pc->flags = PAGE_CGROUP_FLAG_CACHE;
  520. lock_page_cgroup(page);
  521. if (page_get_page_cgroup(page)) {
  522. unlock_page_cgroup(page);
  523. /*
  524. * Another charge has been added to this page already.
  525. * We take lock_page_cgroup(page) again and read
  526. * page->cgroup, increment refcnt.... just retry is OK.
  527. */
  528. res_counter_uncharge(&mem->res, PAGE_SIZE);
  529. css_put(&mem->css);
  530. kmem_cache_free(page_cgroup_cache, pc);
  531. goto retry;
  532. }
  533. page_assign_page_cgroup(page, pc);
  534. mz = page_cgroup_zoneinfo(pc);
  535. spin_lock_irqsave(&mz->lru_lock, flags);
  536. __mem_cgroup_add_list(mz, pc);
  537. spin_unlock_irqrestore(&mz->lru_lock, flags);
  538. unlock_page_cgroup(page);
  539. done:
  540. return 0;
  541. out:
  542. css_put(&mem->css);
  543. kmem_cache_free(page_cgroup_cache, pc);
  544. err:
  545. return -ENOMEM;
  546. }
  547. int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask)
  548. {
  549. return mem_cgroup_charge_common(page, mm, gfp_mask,
  550. MEM_CGROUP_CHARGE_TYPE_MAPPED);
  551. }
  552. int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
  553. gfp_t gfp_mask)
  554. {
  555. if (!mm)
  556. mm = &init_mm;
  557. return mem_cgroup_charge_common(page, mm, gfp_mask,
  558. MEM_CGROUP_CHARGE_TYPE_CACHE);
  559. }
  560. /*
  561. * Uncharging is always a welcome operation, we never complain, simply
  562. * uncharge.
  563. */
  564. void mem_cgroup_uncharge_page(struct page *page)
  565. {
  566. struct page_cgroup *pc;
  567. struct mem_cgroup *mem;
  568. struct mem_cgroup_per_zone *mz;
  569. unsigned long flags;
  570. if (mem_cgroup_subsys.disabled)
  571. return;
  572. /*
  573. * Check if our page_cgroup is valid
  574. */
  575. lock_page_cgroup(page);
  576. pc = page_get_page_cgroup(page);
  577. if (!pc)
  578. goto unlock;
  579. VM_BUG_ON(pc->page != page);
  580. VM_BUG_ON(pc->ref_cnt <= 0);
  581. if (--(pc->ref_cnt) == 0) {
  582. mz = page_cgroup_zoneinfo(pc);
  583. spin_lock_irqsave(&mz->lru_lock, flags);
  584. __mem_cgroup_remove_list(mz, pc);
  585. spin_unlock_irqrestore(&mz->lru_lock, flags);
  586. page_assign_page_cgroup(page, NULL);
  587. unlock_page_cgroup(page);
  588. mem = pc->mem_cgroup;
  589. res_counter_uncharge(&mem->res, PAGE_SIZE);
  590. css_put(&mem->css);
  591. kmem_cache_free(page_cgroup_cache, pc);
  592. return;
  593. }
  594. unlock:
  595. unlock_page_cgroup(page);
  596. }
  597. /*
  598. * Returns non-zero if a page (under migration) has valid page_cgroup member.
  599. * Refcnt of page_cgroup is incremented.
  600. */
  601. int mem_cgroup_prepare_migration(struct page *page)
  602. {
  603. struct page_cgroup *pc;
  604. if (mem_cgroup_subsys.disabled)
  605. return 0;
  606. lock_page_cgroup(page);
  607. pc = page_get_page_cgroup(page);
  608. if (pc)
  609. pc->ref_cnt++;
  610. unlock_page_cgroup(page);
  611. return pc != NULL;
  612. }
  613. void mem_cgroup_end_migration(struct page *page)
  614. {
  615. mem_cgroup_uncharge_page(page);
  616. }
  617. /*
  618. * We know both *page* and *newpage* are now not-on-LRU and PG_locked.
  619. * And no race with uncharge() routines because page_cgroup for *page*
  620. * has extra one reference by mem_cgroup_prepare_migration.
  621. */
  622. void mem_cgroup_page_migration(struct page *page, struct page *newpage)
  623. {
  624. struct page_cgroup *pc;
  625. struct mem_cgroup_per_zone *mz;
  626. unsigned long flags;
  627. lock_page_cgroup(page);
  628. pc = page_get_page_cgroup(page);
  629. if (!pc) {
  630. unlock_page_cgroup(page);
  631. return;
  632. }
  633. mz = page_cgroup_zoneinfo(pc);
  634. spin_lock_irqsave(&mz->lru_lock, flags);
  635. __mem_cgroup_remove_list(mz, pc);
  636. spin_unlock_irqrestore(&mz->lru_lock, flags);
  637. page_assign_page_cgroup(page, NULL);
  638. unlock_page_cgroup(page);
  639. pc->page = newpage;
  640. lock_page_cgroup(newpage);
  641. page_assign_page_cgroup(newpage, pc);
  642. mz = page_cgroup_zoneinfo(pc);
  643. spin_lock_irqsave(&mz->lru_lock, flags);
  644. __mem_cgroup_add_list(mz, pc);
  645. spin_unlock_irqrestore(&mz->lru_lock, flags);
  646. unlock_page_cgroup(newpage);
  647. }
  648. /*
  649. * This routine traverse page_cgroup in given list and drop them all.
  650. * This routine ignores page_cgroup->ref_cnt.
  651. * *And* this routine doesn't reclaim page itself, just removes page_cgroup.
  652. */
  653. #define FORCE_UNCHARGE_BATCH (128)
  654. static void mem_cgroup_force_empty_list(struct mem_cgroup *mem,
  655. struct mem_cgroup_per_zone *mz,
  656. int active)
  657. {
  658. struct page_cgroup *pc;
  659. struct page *page;
  660. int count = FORCE_UNCHARGE_BATCH;
  661. unsigned long flags;
  662. struct list_head *list;
  663. if (active)
  664. list = &mz->active_list;
  665. else
  666. list = &mz->inactive_list;
  667. spin_lock_irqsave(&mz->lru_lock, flags);
  668. while (!list_empty(list)) {
  669. pc = list_entry(list->prev, struct page_cgroup, lru);
  670. page = pc->page;
  671. get_page(page);
  672. spin_unlock_irqrestore(&mz->lru_lock, flags);
  673. mem_cgroup_uncharge_page(page);
  674. put_page(page);
  675. if (--count <= 0) {
  676. count = FORCE_UNCHARGE_BATCH;
  677. cond_resched();
  678. }
  679. spin_lock_irqsave(&mz->lru_lock, flags);
  680. }
  681. spin_unlock_irqrestore(&mz->lru_lock, flags);
  682. }
  683. /*
  684. * make mem_cgroup's charge to be 0 if there is no task.
  685. * This enables deleting this mem_cgroup.
  686. */
  687. static int mem_cgroup_force_empty(struct mem_cgroup *mem)
  688. {
  689. int ret = -EBUSY;
  690. int node, zid;
  691. if (mem_cgroup_subsys.disabled)
  692. return 0;
  693. css_get(&mem->css);
  694. /*
  695. * page reclaim code (kswapd etc..) will move pages between
  696. * active_list <-> inactive_list while we don't take a lock.
  697. * So, we have to do loop here until all lists are empty.
  698. */
  699. while (mem->res.usage > 0) {
  700. if (atomic_read(&mem->css.cgroup->count) > 0)
  701. goto out;
  702. for_each_node_state(node, N_POSSIBLE)
  703. for (zid = 0; zid < MAX_NR_ZONES; zid++) {
  704. struct mem_cgroup_per_zone *mz;
  705. mz = mem_cgroup_zoneinfo(mem, node, zid);
  706. /* drop all page_cgroup in active_list */
  707. mem_cgroup_force_empty_list(mem, mz, 1);
  708. /* drop all page_cgroup in inactive_list */
  709. mem_cgroup_force_empty_list(mem, mz, 0);
  710. }
  711. }
  712. ret = 0;
  713. out:
  714. css_put(&mem->css);
  715. return ret;
  716. }
  717. static int mem_cgroup_write_strategy(char *buf, unsigned long long *tmp)
  718. {
  719. *tmp = memparse(buf, &buf);
  720. if (*buf != '\0')
  721. return -EINVAL;
  722. /*
  723. * Round up the value to the closest page size
  724. */
  725. *tmp = ((*tmp + PAGE_SIZE - 1) >> PAGE_SHIFT) << PAGE_SHIFT;
  726. return 0;
  727. }
  728. static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft)
  729. {
  730. return res_counter_read_u64(&mem_cgroup_from_cont(cont)->res,
  731. cft->private);
  732. }
  733. static ssize_t mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
  734. struct file *file, const char __user *userbuf,
  735. size_t nbytes, loff_t *ppos)
  736. {
  737. return res_counter_write(&mem_cgroup_from_cont(cont)->res,
  738. cft->private, userbuf, nbytes, ppos,
  739. mem_cgroup_write_strategy);
  740. }
  741. static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
  742. {
  743. struct mem_cgroup *mem;
  744. mem = mem_cgroup_from_cont(cont);
  745. switch (event) {
  746. case RES_MAX_USAGE:
  747. res_counter_reset_max(&mem->res);
  748. break;
  749. case RES_FAILCNT:
  750. res_counter_reset_failcnt(&mem->res);
  751. break;
  752. }
  753. return 0;
  754. }
  755. static int mem_force_empty_write(struct cgroup *cont, unsigned int event)
  756. {
  757. return mem_cgroup_force_empty(mem_cgroup_from_cont(cont));
  758. }
  759. static const struct mem_cgroup_stat_desc {
  760. const char *msg;
  761. u64 unit;
  762. } mem_cgroup_stat_desc[] = {
  763. [MEM_CGROUP_STAT_CACHE] = { "cache", PAGE_SIZE, },
  764. [MEM_CGROUP_STAT_RSS] = { "rss", PAGE_SIZE, },
  765. };
  766. static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
  767. struct cgroup_map_cb *cb)
  768. {
  769. struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
  770. struct mem_cgroup_stat *stat = &mem_cont->stat;
  771. int i;
  772. for (i = 0; i < ARRAY_SIZE(stat->cpustat[0].count); i++) {
  773. s64 val;
  774. val = mem_cgroup_read_stat(stat, i);
  775. val *= mem_cgroup_stat_desc[i].unit;
  776. cb->fill(cb, mem_cgroup_stat_desc[i].msg, val);
  777. }
  778. /* showing # of active pages */
  779. {
  780. unsigned long active, inactive;
  781. inactive = mem_cgroup_get_all_zonestat(mem_cont,
  782. MEM_CGROUP_ZSTAT_INACTIVE);
  783. active = mem_cgroup_get_all_zonestat(mem_cont,
  784. MEM_CGROUP_ZSTAT_ACTIVE);
  785. cb->fill(cb, "active", (active) * PAGE_SIZE);
  786. cb->fill(cb, "inactive", (inactive) * PAGE_SIZE);
  787. }
  788. return 0;
  789. }
  790. static struct cftype mem_cgroup_files[] = {
  791. {
  792. .name = "usage_in_bytes",
  793. .private = RES_USAGE,
  794. .read_u64 = mem_cgroup_read,
  795. },
  796. {
  797. .name = "max_usage_in_bytes",
  798. .private = RES_MAX_USAGE,
  799. .trigger = mem_cgroup_reset,
  800. .read_u64 = mem_cgroup_read,
  801. },
  802. {
  803. .name = "limit_in_bytes",
  804. .private = RES_LIMIT,
  805. .write = mem_cgroup_write,
  806. .read_u64 = mem_cgroup_read,
  807. },
  808. {
  809. .name = "failcnt",
  810. .private = RES_FAILCNT,
  811. .trigger = mem_cgroup_reset,
  812. .read_u64 = mem_cgroup_read,
  813. },
  814. {
  815. .name = "force_empty",
  816. .trigger = mem_force_empty_write,
  817. },
  818. {
  819. .name = "stat",
  820. .read_map = mem_control_stat_show,
  821. },
  822. };
  823. static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
  824. {
  825. struct mem_cgroup_per_node *pn;
  826. struct mem_cgroup_per_zone *mz;
  827. int zone, tmp = node;
  828. /*
  829. * This routine is called against possible nodes.
  830. * But it's BUG to call kmalloc() against offline node.
  831. *
  832. * TODO: this routine can waste much memory for nodes which will
  833. * never be onlined. It's better to use memory hotplug callback
  834. * function.
  835. */
  836. if (!node_state(node, N_NORMAL_MEMORY))
  837. tmp = -1;
  838. pn = kmalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
  839. if (!pn)
  840. return 1;
  841. mem->info.nodeinfo[node] = pn;
  842. memset(pn, 0, sizeof(*pn));
  843. for (zone = 0; zone < MAX_NR_ZONES; zone++) {
  844. mz = &pn->zoneinfo[zone];
  845. INIT_LIST_HEAD(&mz->active_list);
  846. INIT_LIST_HEAD(&mz->inactive_list);
  847. spin_lock_init(&mz->lru_lock);
  848. }
  849. return 0;
  850. }
  851. static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
  852. {
  853. kfree(mem->info.nodeinfo[node]);
  854. }
  855. static struct mem_cgroup *mem_cgroup_alloc(void)
  856. {
  857. struct mem_cgroup *mem;
  858. if (sizeof(*mem) < PAGE_SIZE)
  859. mem = kmalloc(sizeof(*mem), GFP_KERNEL);
  860. else
  861. mem = vmalloc(sizeof(*mem));
  862. if (mem)
  863. memset(mem, 0, sizeof(*mem));
  864. return mem;
  865. }
  866. static void mem_cgroup_free(struct mem_cgroup *mem)
  867. {
  868. if (sizeof(*mem) < PAGE_SIZE)
  869. kfree(mem);
  870. else
  871. vfree(mem);
  872. }
  873. static struct cgroup_subsys_state *
  874. mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
  875. {
  876. struct mem_cgroup *mem;
  877. int node;
  878. if (unlikely((cont->parent) == NULL)) {
  879. mem = &init_mem_cgroup;
  880. page_cgroup_cache = KMEM_CACHE(page_cgroup, SLAB_PANIC);
  881. } else {
  882. mem = mem_cgroup_alloc();
  883. if (!mem)
  884. return ERR_PTR(-ENOMEM);
  885. }
  886. res_counter_init(&mem->res);
  887. for_each_node_state(node, N_POSSIBLE)
  888. if (alloc_mem_cgroup_per_zone_info(mem, node))
  889. goto free_out;
  890. return &mem->css;
  891. free_out:
  892. for_each_node_state(node, N_POSSIBLE)
  893. free_mem_cgroup_per_zone_info(mem, node);
  894. if (cont->parent != NULL)
  895. mem_cgroup_free(mem);
  896. return ERR_PTR(-ENOMEM);
  897. }
  898. static void mem_cgroup_pre_destroy(struct cgroup_subsys *ss,
  899. struct cgroup *cont)
  900. {
  901. struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
  902. mem_cgroup_force_empty(mem);
  903. }
  904. static void mem_cgroup_destroy(struct cgroup_subsys *ss,
  905. struct cgroup *cont)
  906. {
  907. int node;
  908. struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
  909. for_each_node_state(node, N_POSSIBLE)
  910. free_mem_cgroup_per_zone_info(mem, node);
  911. mem_cgroup_free(mem_cgroup_from_cont(cont));
  912. }
  913. static int mem_cgroup_populate(struct cgroup_subsys *ss,
  914. struct cgroup *cont)
  915. {
  916. if (mem_cgroup_subsys.disabled)
  917. return 0;
  918. return cgroup_add_files(cont, ss, mem_cgroup_files,
  919. ARRAY_SIZE(mem_cgroup_files));
  920. }
  921. static void mem_cgroup_move_task(struct cgroup_subsys *ss,
  922. struct cgroup *cont,
  923. struct cgroup *old_cont,
  924. struct task_struct *p)
  925. {
  926. struct mm_struct *mm;
  927. struct mem_cgroup *mem, *old_mem;
  928. if (mem_cgroup_subsys.disabled)
  929. return;
  930. mm = get_task_mm(p);
  931. if (mm == NULL)
  932. return;
  933. mem = mem_cgroup_from_cont(cont);
  934. old_mem = mem_cgroup_from_cont(old_cont);
  935. if (mem == old_mem)
  936. goto out;
  937. /*
  938. * Only thread group leaders are allowed to migrate, the mm_struct is
  939. * in effect owned by the leader
  940. */
  941. if (!thread_group_leader(p))
  942. goto out;
  943. out:
  944. mmput(mm);
  945. }
  946. struct cgroup_subsys mem_cgroup_subsys = {
  947. .name = "memory",
  948. .subsys_id = mem_cgroup_subsys_id,
  949. .create = mem_cgroup_create,
  950. .pre_destroy = mem_cgroup_pre_destroy,
  951. .destroy = mem_cgroup_destroy,
  952. .populate = mem_cgroup_populate,
  953. .attach = mem_cgroup_move_task,
  954. .early_init = 0,
  955. };