blk-cgroup.c 46 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825
  1. /*
  2. * Common Block IO controller cgroup interface
  3. *
  4. * Based on ideas and code from CFQ, CFS and BFQ:
  5. * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
  6. *
  7. * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
  8. * Paolo Valente <paolo.valente@unimore.it>
  9. *
  10. * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
  11. * Nauman Rafique <nauman@google.com>
  12. */
  13. #include <linux/ioprio.h>
  14. #include <linux/seq_file.h>
  15. #include <linux/kdev_t.h>
  16. #include <linux/module.h>
  17. #include <linux/err.h>
  18. #include <linux/blkdev.h>
  19. #include <linux/slab.h>
  20. #include <linux/genhd.h>
  21. #include <linux/delay.h>
  22. #include "blk-cgroup.h"
  23. #define MAX_KEY_LEN 100
  24. static DEFINE_SPINLOCK(blkio_list_lock);
  25. static LIST_HEAD(blkio_list);
  26. struct blkio_cgroup blkio_root_cgroup = { .weight = 2*BLKIO_WEIGHT_DEFAULT };
  27. EXPORT_SYMBOL_GPL(blkio_root_cgroup);
  28. static struct blkio_policy_type *blkio_policy[BLKIO_NR_POLICIES];
  29. static struct cgroup_subsys_state *blkiocg_create(struct cgroup_subsys *,
  30. struct cgroup *);
  31. static int blkiocg_can_attach(struct cgroup_subsys *, struct cgroup *,
  32. struct cgroup_taskset *);
  33. static void blkiocg_attach(struct cgroup_subsys *, struct cgroup *,
  34. struct cgroup_taskset *);
  35. static void blkiocg_destroy(struct cgroup_subsys *, struct cgroup *);
  36. static int blkiocg_populate(struct cgroup_subsys *, struct cgroup *);
  37. /* for encoding cft->private value on file */
  38. #define BLKIOFILE_PRIVATE(x, val) (((x) << 16) | (val))
  39. /* What policy owns the file, proportional or throttle */
  40. #define BLKIOFILE_POLICY(val) (((val) >> 16) & 0xffff)
  41. #define BLKIOFILE_ATTR(val) ((val) & 0xffff)
  42. struct cgroup_subsys blkio_subsys = {
  43. .name = "blkio",
  44. .create = blkiocg_create,
  45. .can_attach = blkiocg_can_attach,
  46. .attach = blkiocg_attach,
  47. .destroy = blkiocg_destroy,
  48. .populate = blkiocg_populate,
  49. .subsys_id = blkio_subsys_id,
  50. .use_id = 1,
  51. .module = THIS_MODULE,
  52. };
  53. EXPORT_SYMBOL_GPL(blkio_subsys);
  54. static inline void blkio_policy_insert_node(struct blkio_cgroup *blkcg,
  55. struct blkio_policy_node *pn)
  56. {
  57. list_add(&pn->node, &blkcg->policy_list);
  58. }
  59. static inline bool cftype_blkg_same_policy(struct cftype *cft,
  60. struct blkio_group *blkg)
  61. {
  62. enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
  63. if (blkg->plid == plid)
  64. return 1;
  65. return 0;
  66. }
  67. /* Determines if policy node matches cgroup file being accessed */
  68. static inline bool pn_matches_cftype(struct cftype *cft,
  69. struct blkio_policy_node *pn)
  70. {
  71. enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
  72. int fileid = BLKIOFILE_ATTR(cft->private);
  73. return (plid == pn->plid && fileid == pn->fileid);
  74. }
  75. /* Must be called with blkcg->lock held */
  76. static inline void blkio_policy_delete_node(struct blkio_policy_node *pn)
  77. {
  78. list_del(&pn->node);
  79. }
  80. /* Must be called with blkcg->lock held */
  81. static struct blkio_policy_node *
  82. blkio_policy_search_node(const struct blkio_cgroup *blkcg, dev_t dev,
  83. enum blkio_policy_id plid, int fileid)
  84. {
  85. struct blkio_policy_node *pn;
  86. list_for_each_entry(pn, &blkcg->policy_list, node) {
  87. if (pn->dev == dev && pn->plid == plid && pn->fileid == fileid)
  88. return pn;
  89. }
  90. return NULL;
  91. }
  92. struct blkio_cgroup *cgroup_to_blkio_cgroup(struct cgroup *cgroup)
  93. {
  94. return container_of(cgroup_subsys_state(cgroup, blkio_subsys_id),
  95. struct blkio_cgroup, css);
  96. }
  97. EXPORT_SYMBOL_GPL(cgroup_to_blkio_cgroup);
  98. struct blkio_cgroup *task_blkio_cgroup(struct task_struct *tsk)
  99. {
  100. return container_of(task_subsys_state(tsk, blkio_subsys_id),
  101. struct blkio_cgroup, css);
  102. }
  103. EXPORT_SYMBOL_GPL(task_blkio_cgroup);
  104. static inline void
  105. blkio_update_group_weight(struct blkio_group *blkg, unsigned int weight)
  106. {
  107. struct blkio_policy_type *blkiop;
  108. list_for_each_entry(blkiop, &blkio_list, list) {
  109. /* If this policy does not own the blkg, do not send updates */
  110. if (blkiop->plid != blkg->plid)
  111. continue;
  112. if (blkiop->ops.blkio_update_group_weight_fn)
  113. blkiop->ops.blkio_update_group_weight_fn(blkg->q,
  114. blkg, weight);
  115. }
  116. }
  117. static inline void blkio_update_group_bps(struct blkio_group *blkg, u64 bps,
  118. int fileid)
  119. {
  120. struct blkio_policy_type *blkiop;
  121. list_for_each_entry(blkiop, &blkio_list, list) {
  122. /* If this policy does not own the blkg, do not send updates */
  123. if (blkiop->plid != blkg->plid)
  124. continue;
  125. if (fileid == BLKIO_THROTL_read_bps_device
  126. && blkiop->ops.blkio_update_group_read_bps_fn)
  127. blkiop->ops.blkio_update_group_read_bps_fn(blkg->q,
  128. blkg, bps);
  129. if (fileid == BLKIO_THROTL_write_bps_device
  130. && blkiop->ops.blkio_update_group_write_bps_fn)
  131. blkiop->ops.blkio_update_group_write_bps_fn(blkg->q,
  132. blkg, bps);
  133. }
  134. }
  135. static inline void blkio_update_group_iops(struct blkio_group *blkg,
  136. unsigned int iops, int fileid)
  137. {
  138. struct blkio_policy_type *blkiop;
  139. list_for_each_entry(blkiop, &blkio_list, list) {
  140. /* If this policy does not own the blkg, do not send updates */
  141. if (blkiop->plid != blkg->plid)
  142. continue;
  143. if (fileid == BLKIO_THROTL_read_iops_device
  144. && blkiop->ops.blkio_update_group_read_iops_fn)
  145. blkiop->ops.blkio_update_group_read_iops_fn(blkg->q,
  146. blkg, iops);
  147. if (fileid == BLKIO_THROTL_write_iops_device
  148. && blkiop->ops.blkio_update_group_write_iops_fn)
  149. blkiop->ops.blkio_update_group_write_iops_fn(blkg->q,
  150. blkg,iops);
  151. }
  152. }
  153. /*
  154. * Add to the appropriate stat variable depending on the request type.
  155. * This should be called with the blkg->stats_lock held.
  156. */
  157. static void blkio_add_stat(uint64_t *stat, uint64_t add, bool direction,
  158. bool sync)
  159. {
  160. if (direction)
  161. stat[BLKIO_STAT_WRITE] += add;
  162. else
  163. stat[BLKIO_STAT_READ] += add;
  164. if (sync)
  165. stat[BLKIO_STAT_SYNC] += add;
  166. else
  167. stat[BLKIO_STAT_ASYNC] += add;
  168. }
  169. /*
  170. * Decrements the appropriate stat variable if non-zero depending on the
  171. * request type. Panics on value being zero.
  172. * This should be called with the blkg->stats_lock held.
  173. */
  174. static void blkio_check_and_dec_stat(uint64_t *stat, bool direction, bool sync)
  175. {
  176. if (direction) {
  177. BUG_ON(stat[BLKIO_STAT_WRITE] == 0);
  178. stat[BLKIO_STAT_WRITE]--;
  179. } else {
  180. BUG_ON(stat[BLKIO_STAT_READ] == 0);
  181. stat[BLKIO_STAT_READ]--;
  182. }
  183. if (sync) {
  184. BUG_ON(stat[BLKIO_STAT_SYNC] == 0);
  185. stat[BLKIO_STAT_SYNC]--;
  186. } else {
  187. BUG_ON(stat[BLKIO_STAT_ASYNC] == 0);
  188. stat[BLKIO_STAT_ASYNC]--;
  189. }
  190. }
  191. #ifdef CONFIG_DEBUG_BLK_CGROUP
  192. /* This should be called with the blkg->stats_lock held. */
  193. static void blkio_set_start_group_wait_time(struct blkio_group *blkg,
  194. struct blkio_group *curr_blkg)
  195. {
  196. if (blkio_blkg_waiting(&blkg->stats))
  197. return;
  198. if (blkg == curr_blkg)
  199. return;
  200. blkg->stats.start_group_wait_time = sched_clock();
  201. blkio_mark_blkg_waiting(&blkg->stats);
  202. }
  203. /* This should be called with the blkg->stats_lock held. */
  204. static void blkio_update_group_wait_time(struct blkio_group_stats *stats)
  205. {
  206. unsigned long long now;
  207. if (!blkio_blkg_waiting(stats))
  208. return;
  209. now = sched_clock();
  210. if (time_after64(now, stats->start_group_wait_time))
  211. stats->group_wait_time += now - stats->start_group_wait_time;
  212. blkio_clear_blkg_waiting(stats);
  213. }
  214. /* This should be called with the blkg->stats_lock held. */
  215. static void blkio_end_empty_time(struct blkio_group_stats *stats)
  216. {
  217. unsigned long long now;
  218. if (!blkio_blkg_empty(stats))
  219. return;
  220. now = sched_clock();
  221. if (time_after64(now, stats->start_empty_time))
  222. stats->empty_time += now - stats->start_empty_time;
  223. blkio_clear_blkg_empty(stats);
  224. }
  225. void blkiocg_update_set_idle_time_stats(struct blkio_group *blkg)
  226. {
  227. unsigned long flags;
  228. spin_lock_irqsave(&blkg->stats_lock, flags);
  229. BUG_ON(blkio_blkg_idling(&blkg->stats));
  230. blkg->stats.start_idle_time = sched_clock();
  231. blkio_mark_blkg_idling(&blkg->stats);
  232. spin_unlock_irqrestore(&blkg->stats_lock, flags);
  233. }
  234. EXPORT_SYMBOL_GPL(blkiocg_update_set_idle_time_stats);
  235. void blkiocg_update_idle_time_stats(struct blkio_group *blkg)
  236. {
  237. unsigned long flags;
  238. unsigned long long now;
  239. struct blkio_group_stats *stats;
  240. spin_lock_irqsave(&blkg->stats_lock, flags);
  241. stats = &blkg->stats;
  242. if (blkio_blkg_idling(stats)) {
  243. now = sched_clock();
  244. if (time_after64(now, stats->start_idle_time))
  245. stats->idle_time += now - stats->start_idle_time;
  246. blkio_clear_blkg_idling(stats);
  247. }
  248. spin_unlock_irqrestore(&blkg->stats_lock, flags);
  249. }
  250. EXPORT_SYMBOL_GPL(blkiocg_update_idle_time_stats);
  251. void blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg)
  252. {
  253. unsigned long flags;
  254. struct blkio_group_stats *stats;
  255. spin_lock_irqsave(&blkg->stats_lock, flags);
  256. stats = &blkg->stats;
  257. stats->avg_queue_size_sum +=
  258. stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_READ] +
  259. stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_WRITE];
  260. stats->avg_queue_size_samples++;
  261. blkio_update_group_wait_time(stats);
  262. spin_unlock_irqrestore(&blkg->stats_lock, flags);
  263. }
  264. EXPORT_SYMBOL_GPL(blkiocg_update_avg_queue_size_stats);
  265. void blkiocg_set_start_empty_time(struct blkio_group *blkg)
  266. {
  267. unsigned long flags;
  268. struct blkio_group_stats *stats;
  269. spin_lock_irqsave(&blkg->stats_lock, flags);
  270. stats = &blkg->stats;
  271. if (stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_READ] ||
  272. stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_WRITE]) {
  273. spin_unlock_irqrestore(&blkg->stats_lock, flags);
  274. return;
  275. }
  276. /*
  277. * group is already marked empty. This can happen if cfqq got new
  278. * request in parent group and moved to this group while being added
  279. * to service tree. Just ignore the event and move on.
  280. */
  281. if(blkio_blkg_empty(stats)) {
  282. spin_unlock_irqrestore(&blkg->stats_lock, flags);
  283. return;
  284. }
  285. stats->start_empty_time = sched_clock();
  286. blkio_mark_blkg_empty(stats);
  287. spin_unlock_irqrestore(&blkg->stats_lock, flags);
  288. }
  289. EXPORT_SYMBOL_GPL(blkiocg_set_start_empty_time);
  290. void blkiocg_update_dequeue_stats(struct blkio_group *blkg,
  291. unsigned long dequeue)
  292. {
  293. blkg->stats.dequeue += dequeue;
  294. }
  295. EXPORT_SYMBOL_GPL(blkiocg_update_dequeue_stats);
  296. #else
  297. static inline void blkio_set_start_group_wait_time(struct blkio_group *blkg,
  298. struct blkio_group *curr_blkg) {}
  299. static inline void blkio_end_empty_time(struct blkio_group_stats *stats) {}
  300. #endif
  301. void blkiocg_update_io_add_stats(struct blkio_group *blkg,
  302. struct blkio_group *curr_blkg, bool direction,
  303. bool sync)
  304. {
  305. unsigned long flags;
  306. spin_lock_irqsave(&blkg->stats_lock, flags);
  307. blkio_add_stat(blkg->stats.stat_arr[BLKIO_STAT_QUEUED], 1, direction,
  308. sync);
  309. blkio_end_empty_time(&blkg->stats);
  310. blkio_set_start_group_wait_time(blkg, curr_blkg);
  311. spin_unlock_irqrestore(&blkg->stats_lock, flags);
  312. }
  313. EXPORT_SYMBOL_GPL(blkiocg_update_io_add_stats);
  314. void blkiocg_update_io_remove_stats(struct blkio_group *blkg,
  315. bool direction, bool sync)
  316. {
  317. unsigned long flags;
  318. spin_lock_irqsave(&blkg->stats_lock, flags);
  319. blkio_check_and_dec_stat(blkg->stats.stat_arr[BLKIO_STAT_QUEUED],
  320. direction, sync);
  321. spin_unlock_irqrestore(&blkg->stats_lock, flags);
  322. }
  323. EXPORT_SYMBOL_GPL(blkiocg_update_io_remove_stats);
  324. void blkiocg_update_timeslice_used(struct blkio_group *blkg, unsigned long time,
  325. unsigned long unaccounted_time)
  326. {
  327. unsigned long flags;
  328. spin_lock_irqsave(&blkg->stats_lock, flags);
  329. blkg->stats.time += time;
  330. #ifdef CONFIG_DEBUG_BLK_CGROUP
  331. blkg->stats.unaccounted_time += unaccounted_time;
  332. #endif
  333. spin_unlock_irqrestore(&blkg->stats_lock, flags);
  334. }
  335. EXPORT_SYMBOL_GPL(blkiocg_update_timeslice_used);
  336. /*
  337. * should be called under rcu read lock or queue lock to make sure blkg pointer
  338. * is valid.
  339. */
  340. void blkiocg_update_dispatch_stats(struct blkio_group *blkg,
  341. uint64_t bytes, bool direction, bool sync)
  342. {
  343. struct blkio_group_stats_cpu *stats_cpu;
  344. unsigned long flags;
  345. /*
  346. * Disabling interrupts to provide mutual exclusion between two
  347. * writes on same cpu. It probably is not needed for 64bit. Not
  348. * optimizing that case yet.
  349. */
  350. local_irq_save(flags);
  351. stats_cpu = this_cpu_ptr(blkg->stats_cpu);
  352. u64_stats_update_begin(&stats_cpu->syncp);
  353. stats_cpu->sectors += bytes >> 9;
  354. blkio_add_stat(stats_cpu->stat_arr_cpu[BLKIO_STAT_CPU_SERVICED],
  355. 1, direction, sync);
  356. blkio_add_stat(stats_cpu->stat_arr_cpu[BLKIO_STAT_CPU_SERVICE_BYTES],
  357. bytes, direction, sync);
  358. u64_stats_update_end(&stats_cpu->syncp);
  359. local_irq_restore(flags);
  360. }
  361. EXPORT_SYMBOL_GPL(blkiocg_update_dispatch_stats);
  362. void blkiocg_update_completion_stats(struct blkio_group *blkg,
  363. uint64_t start_time, uint64_t io_start_time, bool direction, bool sync)
  364. {
  365. struct blkio_group_stats *stats;
  366. unsigned long flags;
  367. unsigned long long now = sched_clock();
  368. spin_lock_irqsave(&blkg->stats_lock, flags);
  369. stats = &blkg->stats;
  370. if (time_after64(now, io_start_time))
  371. blkio_add_stat(stats->stat_arr[BLKIO_STAT_SERVICE_TIME],
  372. now - io_start_time, direction, sync);
  373. if (time_after64(io_start_time, start_time))
  374. blkio_add_stat(stats->stat_arr[BLKIO_STAT_WAIT_TIME],
  375. io_start_time - start_time, direction, sync);
  376. spin_unlock_irqrestore(&blkg->stats_lock, flags);
  377. }
  378. EXPORT_SYMBOL_GPL(blkiocg_update_completion_stats);
  379. /* Merged stats are per cpu. */
  380. void blkiocg_update_io_merged_stats(struct blkio_group *blkg, bool direction,
  381. bool sync)
  382. {
  383. struct blkio_group_stats_cpu *stats_cpu;
  384. unsigned long flags;
  385. /*
  386. * Disabling interrupts to provide mutual exclusion between two
  387. * writes on same cpu. It probably is not needed for 64bit. Not
  388. * optimizing that case yet.
  389. */
  390. local_irq_save(flags);
  391. stats_cpu = this_cpu_ptr(blkg->stats_cpu);
  392. u64_stats_update_begin(&stats_cpu->syncp);
  393. blkio_add_stat(stats_cpu->stat_arr_cpu[BLKIO_STAT_CPU_MERGED], 1,
  394. direction, sync);
  395. u64_stats_update_end(&stats_cpu->syncp);
  396. local_irq_restore(flags);
  397. }
  398. EXPORT_SYMBOL_GPL(blkiocg_update_io_merged_stats);
  399. struct blkio_group *blkg_lookup_create(struct blkio_cgroup *blkcg,
  400. struct request_queue *q,
  401. enum blkio_policy_id plid,
  402. bool for_root)
  403. __releases(q->queue_lock) __acquires(q->queue_lock)
  404. {
  405. struct blkio_policy_type *pol = blkio_policy[plid];
  406. struct blkio_group *blkg, *new_blkg;
  407. WARN_ON_ONCE(!rcu_read_lock_held());
  408. lockdep_assert_held(q->queue_lock);
  409. /*
  410. * This could be the first entry point of blkcg implementation and
  411. * we shouldn't allow anything to go through for a bypassing queue.
  412. * The following can be removed if blkg lookup is guaranteed to
  413. * fail on a bypassing queue.
  414. */
  415. if (unlikely(blk_queue_bypass(q)) && !for_root)
  416. return ERR_PTR(blk_queue_dead(q) ? -EINVAL : -EBUSY);
  417. blkg = blkg_lookup(blkcg, q, plid);
  418. if (blkg)
  419. return blkg;
  420. if (!css_tryget(&blkcg->css))
  421. return ERR_PTR(-EINVAL);
  422. /*
  423. * Allocate and initialize.
  424. *
  425. * FIXME: The following is broken. Percpu memory allocation
  426. * requires %GFP_KERNEL context and can't be performed from IO
  427. * path. Allocation here should inherently be atomic and the
  428. * following lock dancing can be removed once the broken percpu
  429. * allocation is fixed.
  430. */
  431. spin_unlock_irq(q->queue_lock);
  432. rcu_read_unlock();
  433. new_blkg = pol->ops.blkio_alloc_group_fn(q, blkcg);
  434. if (new_blkg) {
  435. new_blkg->stats_cpu = alloc_percpu(struct blkio_group_stats_cpu);
  436. spin_lock_init(&new_blkg->stats_lock);
  437. rcu_assign_pointer(new_blkg->q, q);
  438. new_blkg->blkcg_id = css_id(&blkcg->css);
  439. new_blkg->plid = plid;
  440. cgroup_path(blkcg->css.cgroup, new_blkg->path,
  441. sizeof(new_blkg->path));
  442. }
  443. rcu_read_lock();
  444. spin_lock_irq(q->queue_lock);
  445. css_put(&blkcg->css);
  446. /* did bypass get turned on inbetween? */
  447. if (unlikely(blk_queue_bypass(q)) && !for_root) {
  448. blkg = ERR_PTR(blk_queue_dead(q) ? -EINVAL : -EBUSY);
  449. goto out;
  450. }
  451. /* did someone beat us to it? */
  452. blkg = blkg_lookup(blkcg, q, plid);
  453. if (unlikely(blkg))
  454. goto out;
  455. /* did alloc fail? */
  456. if (unlikely(!new_blkg || !new_blkg->stats_cpu)) {
  457. blkg = ERR_PTR(-ENOMEM);
  458. goto out;
  459. }
  460. /* insert */
  461. spin_lock(&blkcg->lock);
  462. swap(blkg, new_blkg);
  463. hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
  464. pol->ops.blkio_link_group_fn(q, blkg);
  465. spin_unlock(&blkcg->lock);
  466. out:
  467. if (new_blkg) {
  468. free_percpu(new_blkg->stats_cpu);
  469. kfree(new_blkg);
  470. }
  471. return blkg;
  472. }
  473. EXPORT_SYMBOL_GPL(blkg_lookup_create);
  474. static void __blkiocg_del_blkio_group(struct blkio_group *blkg)
  475. {
  476. hlist_del_init_rcu(&blkg->blkcg_node);
  477. blkg->blkcg_id = 0;
  478. }
  479. /*
  480. * returns 0 if blkio_group was still on cgroup list. Otherwise returns 1
  481. * indicating that blk_group was unhashed by the time we got to it.
  482. */
  483. int blkiocg_del_blkio_group(struct blkio_group *blkg)
  484. {
  485. struct blkio_cgroup *blkcg;
  486. unsigned long flags;
  487. struct cgroup_subsys_state *css;
  488. int ret = 1;
  489. rcu_read_lock();
  490. css = css_lookup(&blkio_subsys, blkg->blkcg_id);
  491. if (css) {
  492. blkcg = container_of(css, struct blkio_cgroup, css);
  493. spin_lock_irqsave(&blkcg->lock, flags);
  494. if (!hlist_unhashed(&blkg->blkcg_node)) {
  495. __blkiocg_del_blkio_group(blkg);
  496. ret = 0;
  497. }
  498. spin_unlock_irqrestore(&blkcg->lock, flags);
  499. }
  500. rcu_read_unlock();
  501. return ret;
  502. }
  503. EXPORT_SYMBOL_GPL(blkiocg_del_blkio_group);
  504. /* called under rcu_read_lock(). */
  505. struct blkio_group *blkg_lookup(struct blkio_cgroup *blkcg,
  506. struct request_queue *q,
  507. enum blkio_policy_id plid)
  508. {
  509. struct blkio_group *blkg;
  510. struct hlist_node *n;
  511. hlist_for_each_entry_rcu(blkg, n, &blkcg->blkg_list, blkcg_node)
  512. if (blkg->q == q && blkg->plid == plid)
  513. return blkg;
  514. return NULL;
  515. }
  516. EXPORT_SYMBOL_GPL(blkg_lookup);
  517. void blkg_destroy_all(struct request_queue *q)
  518. {
  519. struct blkio_policy_type *pol;
  520. while (true) {
  521. bool done = true;
  522. spin_lock(&blkio_list_lock);
  523. spin_lock_irq(q->queue_lock);
  524. /*
  525. * clear_queue_fn() might return with non-empty group list
  526. * if it raced cgroup removal and lost. cgroup removal is
  527. * guaranteed to make forward progress and retrying after a
  528. * while is enough. This ugliness is scheduled to be
  529. * removed after locking update.
  530. */
  531. list_for_each_entry(pol, &blkio_list, list)
  532. if (!pol->ops.blkio_clear_queue_fn(q))
  533. done = false;
  534. spin_unlock_irq(q->queue_lock);
  535. spin_unlock(&blkio_list_lock);
  536. if (done)
  537. break;
  538. msleep(10); /* just some random duration I like */
  539. }
  540. }
  541. static void blkio_reset_stats_cpu(struct blkio_group *blkg)
  542. {
  543. struct blkio_group_stats_cpu *stats_cpu;
  544. int i, j, k;
  545. /*
  546. * Note: On 64 bit arch this should not be an issue. This has the
  547. * possibility of returning some inconsistent value on 32bit arch
  548. * as 64bit update on 32bit is non atomic. Taking care of this
  549. * corner case makes code very complicated, like sending IPIs to
  550. * cpus, taking care of stats of offline cpus etc.
  551. *
  552. * reset stats is anyway more of a debug feature and this sounds a
  553. * corner case. So I am not complicating the code yet until and
  554. * unless this becomes a real issue.
  555. */
  556. for_each_possible_cpu(i) {
  557. stats_cpu = per_cpu_ptr(blkg->stats_cpu, i);
  558. stats_cpu->sectors = 0;
  559. for(j = 0; j < BLKIO_STAT_CPU_NR; j++)
  560. for (k = 0; k < BLKIO_STAT_TOTAL; k++)
  561. stats_cpu->stat_arr_cpu[j][k] = 0;
  562. }
  563. }
  564. static int
  565. blkiocg_reset_stats(struct cgroup *cgroup, struct cftype *cftype, u64 val)
  566. {
  567. struct blkio_cgroup *blkcg;
  568. struct blkio_group *blkg;
  569. struct blkio_group_stats *stats;
  570. struct hlist_node *n;
  571. uint64_t queued[BLKIO_STAT_TOTAL];
  572. int i;
  573. #ifdef CONFIG_DEBUG_BLK_CGROUP
  574. bool idling, waiting, empty;
  575. unsigned long long now = sched_clock();
  576. #endif
  577. blkcg = cgroup_to_blkio_cgroup(cgroup);
  578. spin_lock_irq(&blkcg->lock);
  579. hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
  580. spin_lock(&blkg->stats_lock);
  581. stats = &blkg->stats;
  582. #ifdef CONFIG_DEBUG_BLK_CGROUP
  583. idling = blkio_blkg_idling(stats);
  584. waiting = blkio_blkg_waiting(stats);
  585. empty = blkio_blkg_empty(stats);
  586. #endif
  587. for (i = 0; i < BLKIO_STAT_TOTAL; i++)
  588. queued[i] = stats->stat_arr[BLKIO_STAT_QUEUED][i];
  589. memset(stats, 0, sizeof(struct blkio_group_stats));
  590. for (i = 0; i < BLKIO_STAT_TOTAL; i++)
  591. stats->stat_arr[BLKIO_STAT_QUEUED][i] = queued[i];
  592. #ifdef CONFIG_DEBUG_BLK_CGROUP
  593. if (idling) {
  594. blkio_mark_blkg_idling(stats);
  595. stats->start_idle_time = now;
  596. }
  597. if (waiting) {
  598. blkio_mark_blkg_waiting(stats);
  599. stats->start_group_wait_time = now;
  600. }
  601. if (empty) {
  602. blkio_mark_blkg_empty(stats);
  603. stats->start_empty_time = now;
  604. }
  605. #endif
  606. spin_unlock(&blkg->stats_lock);
  607. /* Reset Per cpu stats which don't take blkg->stats_lock */
  608. blkio_reset_stats_cpu(blkg);
  609. }
  610. spin_unlock_irq(&blkcg->lock);
  611. return 0;
  612. }
  613. static void blkio_get_key_name(enum stat_sub_type type, dev_t dev, char *str,
  614. int chars_left, bool diskname_only)
  615. {
  616. snprintf(str, chars_left, "%d:%d", MAJOR(dev), MINOR(dev));
  617. chars_left -= strlen(str);
  618. if (chars_left <= 0) {
  619. printk(KERN_WARNING
  620. "Possibly incorrect cgroup stat display format");
  621. return;
  622. }
  623. if (diskname_only)
  624. return;
  625. switch (type) {
  626. case BLKIO_STAT_READ:
  627. strlcat(str, " Read", chars_left);
  628. break;
  629. case BLKIO_STAT_WRITE:
  630. strlcat(str, " Write", chars_left);
  631. break;
  632. case BLKIO_STAT_SYNC:
  633. strlcat(str, " Sync", chars_left);
  634. break;
  635. case BLKIO_STAT_ASYNC:
  636. strlcat(str, " Async", chars_left);
  637. break;
  638. case BLKIO_STAT_TOTAL:
  639. strlcat(str, " Total", chars_left);
  640. break;
  641. default:
  642. strlcat(str, " Invalid", chars_left);
  643. }
  644. }
  645. static uint64_t blkio_fill_stat(char *str, int chars_left, uint64_t val,
  646. struct cgroup_map_cb *cb, dev_t dev)
  647. {
  648. blkio_get_key_name(0, dev, str, chars_left, true);
  649. cb->fill(cb, str, val);
  650. return val;
  651. }
  652. static uint64_t blkio_read_stat_cpu(struct blkio_group *blkg,
  653. enum stat_type_cpu type, enum stat_sub_type sub_type)
  654. {
  655. int cpu;
  656. struct blkio_group_stats_cpu *stats_cpu;
  657. u64 val = 0, tval;
  658. for_each_possible_cpu(cpu) {
  659. unsigned int start;
  660. stats_cpu = per_cpu_ptr(blkg->stats_cpu, cpu);
  661. do {
  662. start = u64_stats_fetch_begin(&stats_cpu->syncp);
  663. if (type == BLKIO_STAT_CPU_SECTORS)
  664. tval = stats_cpu->sectors;
  665. else
  666. tval = stats_cpu->stat_arr_cpu[type][sub_type];
  667. } while(u64_stats_fetch_retry(&stats_cpu->syncp, start));
  668. val += tval;
  669. }
  670. return val;
  671. }
  672. static uint64_t blkio_get_stat_cpu(struct blkio_group *blkg,
  673. struct cgroup_map_cb *cb, dev_t dev, enum stat_type_cpu type)
  674. {
  675. uint64_t disk_total, val;
  676. char key_str[MAX_KEY_LEN];
  677. enum stat_sub_type sub_type;
  678. if (type == BLKIO_STAT_CPU_SECTORS) {
  679. val = blkio_read_stat_cpu(blkg, type, 0);
  680. return blkio_fill_stat(key_str, MAX_KEY_LEN - 1, val, cb, dev);
  681. }
  682. for (sub_type = BLKIO_STAT_READ; sub_type < BLKIO_STAT_TOTAL;
  683. sub_type++) {
  684. blkio_get_key_name(sub_type, dev, key_str, MAX_KEY_LEN, false);
  685. val = blkio_read_stat_cpu(blkg, type, sub_type);
  686. cb->fill(cb, key_str, val);
  687. }
  688. disk_total = blkio_read_stat_cpu(blkg, type, BLKIO_STAT_READ) +
  689. blkio_read_stat_cpu(blkg, type, BLKIO_STAT_WRITE);
  690. blkio_get_key_name(BLKIO_STAT_TOTAL, dev, key_str, MAX_KEY_LEN, false);
  691. cb->fill(cb, key_str, disk_total);
  692. return disk_total;
  693. }
  694. /* This should be called with blkg->stats_lock held */
  695. static uint64_t blkio_get_stat(struct blkio_group *blkg,
  696. struct cgroup_map_cb *cb, dev_t dev, enum stat_type type)
  697. {
  698. uint64_t disk_total;
  699. char key_str[MAX_KEY_LEN];
  700. enum stat_sub_type sub_type;
  701. if (type == BLKIO_STAT_TIME)
  702. return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
  703. blkg->stats.time, cb, dev);
  704. #ifdef CONFIG_DEBUG_BLK_CGROUP
  705. if (type == BLKIO_STAT_UNACCOUNTED_TIME)
  706. return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
  707. blkg->stats.unaccounted_time, cb, dev);
  708. if (type == BLKIO_STAT_AVG_QUEUE_SIZE) {
  709. uint64_t sum = blkg->stats.avg_queue_size_sum;
  710. uint64_t samples = blkg->stats.avg_queue_size_samples;
  711. if (samples)
  712. do_div(sum, samples);
  713. else
  714. sum = 0;
  715. return blkio_fill_stat(key_str, MAX_KEY_LEN - 1, sum, cb, dev);
  716. }
  717. if (type == BLKIO_STAT_GROUP_WAIT_TIME)
  718. return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
  719. blkg->stats.group_wait_time, cb, dev);
  720. if (type == BLKIO_STAT_IDLE_TIME)
  721. return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
  722. blkg->stats.idle_time, cb, dev);
  723. if (type == BLKIO_STAT_EMPTY_TIME)
  724. return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
  725. blkg->stats.empty_time, cb, dev);
  726. if (type == BLKIO_STAT_DEQUEUE)
  727. return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
  728. blkg->stats.dequeue, cb, dev);
  729. #endif
  730. for (sub_type = BLKIO_STAT_READ; sub_type < BLKIO_STAT_TOTAL;
  731. sub_type++) {
  732. blkio_get_key_name(sub_type, dev, key_str, MAX_KEY_LEN, false);
  733. cb->fill(cb, key_str, blkg->stats.stat_arr[type][sub_type]);
  734. }
  735. disk_total = blkg->stats.stat_arr[type][BLKIO_STAT_READ] +
  736. blkg->stats.stat_arr[type][BLKIO_STAT_WRITE];
  737. blkio_get_key_name(BLKIO_STAT_TOTAL, dev, key_str, MAX_KEY_LEN, false);
  738. cb->fill(cb, key_str, disk_total);
  739. return disk_total;
  740. }
  741. static int blkio_policy_parse_and_set(char *buf,
  742. struct blkio_policy_node *newpn,
  743. enum blkio_policy_id plid, int fileid,
  744. struct blkio_cgroup *blkcg)
  745. {
  746. struct gendisk *disk = NULL;
  747. struct blkio_group *blkg = NULL;
  748. char *s[4], *p, *major_s = NULL, *minor_s = NULL;
  749. unsigned long major, minor;
  750. int i = 0, ret = -EINVAL;
  751. int part;
  752. dev_t dev;
  753. u64 temp;
  754. memset(s, 0, sizeof(s));
  755. while ((p = strsep(&buf, " ")) != NULL) {
  756. if (!*p)
  757. continue;
  758. s[i++] = p;
  759. /* Prevent from inputing too many things */
  760. if (i == 3)
  761. break;
  762. }
  763. if (i != 2)
  764. goto out;
  765. p = strsep(&s[0], ":");
  766. if (p != NULL)
  767. major_s = p;
  768. else
  769. goto out;
  770. minor_s = s[0];
  771. if (!minor_s)
  772. goto out;
  773. if (strict_strtoul(major_s, 10, &major))
  774. goto out;
  775. if (strict_strtoul(minor_s, 10, &minor))
  776. goto out;
  777. dev = MKDEV(major, minor);
  778. if (strict_strtoull(s[1], 10, &temp))
  779. goto out;
  780. /* For rule removal, do not check for device presence. */
  781. disk = get_gendisk(dev, &part);
  782. if ((!disk || part) && temp) {
  783. ret = -ENODEV;
  784. goto out;
  785. }
  786. rcu_read_lock();
  787. if (disk && !part) {
  788. spin_lock_irq(disk->queue->queue_lock);
  789. blkg = blkg_lookup_create(blkcg, disk->queue, plid, false);
  790. spin_unlock_irq(disk->queue->queue_lock);
  791. if (IS_ERR(blkg)) {
  792. ret = PTR_ERR(blkg);
  793. if (ret == -EBUSY)
  794. goto out_unlock;
  795. blkg = NULL;
  796. }
  797. }
  798. newpn->dev = dev;
  799. switch (plid) {
  800. case BLKIO_POLICY_PROP:
  801. if ((temp < BLKIO_WEIGHT_MIN && temp > 0) ||
  802. temp > BLKIO_WEIGHT_MAX)
  803. goto out_unlock;
  804. newpn->plid = plid;
  805. newpn->fileid = fileid;
  806. newpn->val.weight = temp;
  807. if (blkg)
  808. blkg->conf.weight = temp;
  809. break;
  810. case BLKIO_POLICY_THROTL:
  811. switch(fileid) {
  812. case BLKIO_THROTL_read_bps_device:
  813. if (blkg)
  814. blkg->conf.bps[READ] = temp;
  815. newpn->plid = plid;
  816. newpn->fileid = fileid;
  817. newpn->val.bps = temp;
  818. break;
  819. case BLKIO_THROTL_write_bps_device:
  820. if (blkg)
  821. blkg->conf.bps[WRITE] = temp;
  822. newpn->plid = plid;
  823. newpn->fileid = fileid;
  824. newpn->val.bps = temp;
  825. break;
  826. case BLKIO_THROTL_read_iops_device:
  827. if (temp > THROTL_IOPS_MAX)
  828. goto out_unlock;
  829. if (blkg)
  830. blkg->conf.iops[READ] = temp;
  831. newpn->plid = plid;
  832. newpn->fileid = fileid;
  833. newpn->val.iops = (unsigned int)temp;
  834. break;
  835. case BLKIO_THROTL_write_iops_device:
  836. if (temp > THROTL_IOPS_MAX)
  837. goto out_unlock;
  838. if (blkg)
  839. blkg->conf.iops[WRITE] = temp;
  840. newpn->plid = plid;
  841. newpn->fileid = fileid;
  842. newpn->val.iops = (unsigned int)temp;
  843. break;
  844. }
  845. break;
  846. default:
  847. BUG();
  848. }
  849. ret = 0;
  850. out_unlock:
  851. rcu_read_unlock();
  852. out:
  853. put_disk(disk);
  854. /*
  855. * If queue was bypassing, we should retry. Do so after a short
  856. * msleep(). It isn't strictly necessary but queue can be
  857. * bypassing for some time and it's always nice to avoid busy
  858. * looping.
  859. */
  860. if (ret == -EBUSY) {
  861. msleep(10);
  862. return restart_syscall();
  863. }
  864. return ret;
  865. }
  866. unsigned int blkcg_get_weight(struct blkio_cgroup *blkcg,
  867. dev_t dev)
  868. {
  869. struct blkio_policy_node *pn;
  870. unsigned long flags;
  871. unsigned int weight;
  872. spin_lock_irqsave(&blkcg->lock, flags);
  873. pn = blkio_policy_search_node(blkcg, dev, BLKIO_POLICY_PROP,
  874. BLKIO_PROP_weight_device);
  875. if (pn)
  876. weight = pn->val.weight;
  877. else
  878. weight = blkcg->weight;
  879. spin_unlock_irqrestore(&blkcg->lock, flags);
  880. return weight;
  881. }
  882. EXPORT_SYMBOL_GPL(blkcg_get_weight);
  883. uint64_t blkcg_get_read_bps(struct blkio_cgroup *blkcg, dev_t dev)
  884. {
  885. struct blkio_policy_node *pn;
  886. unsigned long flags;
  887. uint64_t bps = -1;
  888. spin_lock_irqsave(&blkcg->lock, flags);
  889. pn = blkio_policy_search_node(blkcg, dev, BLKIO_POLICY_THROTL,
  890. BLKIO_THROTL_read_bps_device);
  891. if (pn)
  892. bps = pn->val.bps;
  893. spin_unlock_irqrestore(&blkcg->lock, flags);
  894. return bps;
  895. }
  896. uint64_t blkcg_get_write_bps(struct blkio_cgroup *blkcg, dev_t dev)
  897. {
  898. struct blkio_policy_node *pn;
  899. unsigned long flags;
  900. uint64_t bps = -1;
  901. spin_lock_irqsave(&blkcg->lock, flags);
  902. pn = blkio_policy_search_node(blkcg, dev, BLKIO_POLICY_THROTL,
  903. BLKIO_THROTL_write_bps_device);
  904. if (pn)
  905. bps = pn->val.bps;
  906. spin_unlock_irqrestore(&blkcg->lock, flags);
  907. return bps;
  908. }
  909. unsigned int blkcg_get_read_iops(struct blkio_cgroup *blkcg, dev_t dev)
  910. {
  911. struct blkio_policy_node *pn;
  912. unsigned long flags;
  913. unsigned int iops = -1;
  914. spin_lock_irqsave(&blkcg->lock, flags);
  915. pn = blkio_policy_search_node(blkcg, dev, BLKIO_POLICY_THROTL,
  916. BLKIO_THROTL_read_iops_device);
  917. if (pn)
  918. iops = pn->val.iops;
  919. spin_unlock_irqrestore(&blkcg->lock, flags);
  920. return iops;
  921. }
  922. unsigned int blkcg_get_write_iops(struct blkio_cgroup *blkcg, dev_t dev)
  923. {
  924. struct blkio_policy_node *pn;
  925. unsigned long flags;
  926. unsigned int iops = -1;
  927. spin_lock_irqsave(&blkcg->lock, flags);
  928. pn = blkio_policy_search_node(blkcg, dev, BLKIO_POLICY_THROTL,
  929. BLKIO_THROTL_write_iops_device);
  930. if (pn)
  931. iops = pn->val.iops;
  932. spin_unlock_irqrestore(&blkcg->lock, flags);
  933. return iops;
  934. }
  935. /* Checks whether user asked for deleting a policy rule */
  936. static bool blkio_delete_rule_command(struct blkio_policy_node *pn)
  937. {
  938. switch(pn->plid) {
  939. case BLKIO_POLICY_PROP:
  940. if (pn->val.weight == 0)
  941. return 1;
  942. break;
  943. case BLKIO_POLICY_THROTL:
  944. switch(pn->fileid) {
  945. case BLKIO_THROTL_read_bps_device:
  946. case BLKIO_THROTL_write_bps_device:
  947. if (pn->val.bps == 0)
  948. return 1;
  949. break;
  950. case BLKIO_THROTL_read_iops_device:
  951. case BLKIO_THROTL_write_iops_device:
  952. if (pn->val.iops == 0)
  953. return 1;
  954. }
  955. break;
  956. default:
  957. BUG();
  958. }
  959. return 0;
  960. }
  961. static void blkio_update_policy_rule(struct blkio_policy_node *oldpn,
  962. struct blkio_policy_node *newpn)
  963. {
  964. switch(oldpn->plid) {
  965. case BLKIO_POLICY_PROP:
  966. oldpn->val.weight = newpn->val.weight;
  967. break;
  968. case BLKIO_POLICY_THROTL:
  969. switch(newpn->fileid) {
  970. case BLKIO_THROTL_read_bps_device:
  971. case BLKIO_THROTL_write_bps_device:
  972. oldpn->val.bps = newpn->val.bps;
  973. break;
  974. case BLKIO_THROTL_read_iops_device:
  975. case BLKIO_THROTL_write_iops_device:
  976. oldpn->val.iops = newpn->val.iops;
  977. }
  978. break;
  979. default:
  980. BUG();
  981. }
  982. }
  983. /*
  984. * Some rules/values in blkg have changed. Propagate those to respective
  985. * policies.
  986. */
  987. static void blkio_update_blkg_policy(struct blkio_cgroup *blkcg,
  988. struct blkio_group *blkg, struct blkio_policy_node *pn)
  989. {
  990. struct blkio_group_conf *conf = &blkg->conf;
  991. switch(pn->plid) {
  992. case BLKIO_POLICY_PROP:
  993. blkio_update_group_weight(blkg, conf->weight ?: blkcg->weight);
  994. break;
  995. case BLKIO_POLICY_THROTL:
  996. switch(pn->fileid) {
  997. case BLKIO_THROTL_read_bps_device:
  998. blkio_update_group_bps(blkg, conf->bps[READ] ?: -1,
  999. pn->fileid);
  1000. break;
  1001. case BLKIO_THROTL_write_bps_device:
  1002. blkio_update_group_bps(blkg, conf->bps[WRITE] ?: -1,
  1003. pn->fileid);
  1004. break;
  1005. case BLKIO_THROTL_read_iops_device:
  1006. blkio_update_group_iops(blkg, conf->iops[READ] ?: -1,
  1007. pn->fileid);
  1008. break;
  1009. case BLKIO_THROTL_write_iops_device:
  1010. blkio_update_group_iops(blkg, conf->iops[WRITE] ?: -1,
  1011. pn->fileid);
  1012. break;
  1013. }
  1014. break;
  1015. default:
  1016. BUG();
  1017. }
  1018. }
  1019. /*
  1020. * A policy node rule has been updated. Propagate this update to all the
  1021. * block groups which might be affected by this update.
  1022. */
  1023. static void blkio_update_policy_node_blkg(struct blkio_cgroup *blkcg,
  1024. struct blkio_policy_node *pn)
  1025. {
  1026. struct blkio_group *blkg;
  1027. struct hlist_node *n;
  1028. spin_lock(&blkio_list_lock);
  1029. spin_lock_irq(&blkcg->lock);
  1030. hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
  1031. if (pn->dev != blkg->dev || pn->plid != blkg->plid)
  1032. continue;
  1033. blkio_update_blkg_policy(blkcg, blkg, pn);
  1034. }
  1035. spin_unlock_irq(&blkcg->lock);
  1036. spin_unlock(&blkio_list_lock);
  1037. }
  1038. static int blkiocg_file_write(struct cgroup *cgrp, struct cftype *cft,
  1039. const char *buffer)
  1040. {
  1041. int ret = 0;
  1042. char *buf;
  1043. struct blkio_policy_node *newpn, *pn;
  1044. struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);
  1045. int keep_newpn = 0;
  1046. enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
  1047. int fileid = BLKIOFILE_ATTR(cft->private);
  1048. buf = kstrdup(buffer, GFP_KERNEL);
  1049. if (!buf)
  1050. return -ENOMEM;
  1051. newpn = kzalloc(sizeof(*newpn), GFP_KERNEL);
  1052. if (!newpn) {
  1053. ret = -ENOMEM;
  1054. goto free_buf;
  1055. }
  1056. ret = blkio_policy_parse_and_set(buf, newpn, plid, fileid, blkcg);
  1057. if (ret)
  1058. goto free_newpn;
  1059. spin_lock_irq(&blkcg->lock);
  1060. pn = blkio_policy_search_node(blkcg, newpn->dev, plid, fileid);
  1061. if (!pn) {
  1062. if (!blkio_delete_rule_command(newpn)) {
  1063. blkio_policy_insert_node(blkcg, newpn);
  1064. keep_newpn = 1;
  1065. }
  1066. spin_unlock_irq(&blkcg->lock);
  1067. goto update_io_group;
  1068. }
  1069. if (blkio_delete_rule_command(newpn)) {
  1070. blkio_policy_delete_node(pn);
  1071. kfree(pn);
  1072. spin_unlock_irq(&blkcg->lock);
  1073. goto update_io_group;
  1074. }
  1075. spin_unlock_irq(&blkcg->lock);
  1076. blkio_update_policy_rule(pn, newpn);
  1077. update_io_group:
  1078. blkio_update_policy_node_blkg(blkcg, newpn);
  1079. free_newpn:
  1080. if (!keep_newpn)
  1081. kfree(newpn);
  1082. free_buf:
  1083. kfree(buf);
  1084. return ret;
  1085. }
  1086. static void
  1087. blkio_print_policy_node(struct seq_file *m, struct blkio_policy_node *pn)
  1088. {
  1089. switch(pn->plid) {
  1090. case BLKIO_POLICY_PROP:
  1091. if (pn->fileid == BLKIO_PROP_weight_device)
  1092. seq_printf(m, "%u:%u\t%u\n", MAJOR(pn->dev),
  1093. MINOR(pn->dev), pn->val.weight);
  1094. break;
  1095. case BLKIO_POLICY_THROTL:
  1096. switch(pn->fileid) {
  1097. case BLKIO_THROTL_read_bps_device:
  1098. case BLKIO_THROTL_write_bps_device:
  1099. seq_printf(m, "%u:%u\t%llu\n", MAJOR(pn->dev),
  1100. MINOR(pn->dev), pn->val.bps);
  1101. break;
  1102. case BLKIO_THROTL_read_iops_device:
  1103. case BLKIO_THROTL_write_iops_device:
  1104. seq_printf(m, "%u:%u\t%u\n", MAJOR(pn->dev),
  1105. MINOR(pn->dev), pn->val.iops);
  1106. break;
  1107. }
  1108. break;
  1109. default:
  1110. BUG();
  1111. }
  1112. }
  1113. /* cgroup files which read their data from policy nodes end up here */
  1114. static void blkio_read_policy_node_files(struct cftype *cft,
  1115. struct blkio_cgroup *blkcg, struct seq_file *m)
  1116. {
  1117. struct blkio_policy_node *pn;
  1118. if (!list_empty(&blkcg->policy_list)) {
  1119. spin_lock_irq(&blkcg->lock);
  1120. list_for_each_entry(pn, &blkcg->policy_list, node) {
  1121. if (!pn_matches_cftype(cft, pn))
  1122. continue;
  1123. blkio_print_policy_node(m, pn);
  1124. }
  1125. spin_unlock_irq(&blkcg->lock);
  1126. }
  1127. }
  1128. static int blkiocg_file_read(struct cgroup *cgrp, struct cftype *cft,
  1129. struct seq_file *m)
  1130. {
  1131. struct blkio_cgroup *blkcg;
  1132. enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
  1133. int name = BLKIOFILE_ATTR(cft->private);
  1134. blkcg = cgroup_to_blkio_cgroup(cgrp);
  1135. switch(plid) {
  1136. case BLKIO_POLICY_PROP:
  1137. switch(name) {
  1138. case BLKIO_PROP_weight_device:
  1139. blkio_read_policy_node_files(cft, blkcg, m);
  1140. return 0;
  1141. default:
  1142. BUG();
  1143. }
  1144. break;
  1145. case BLKIO_POLICY_THROTL:
  1146. switch(name){
  1147. case BLKIO_THROTL_read_bps_device:
  1148. case BLKIO_THROTL_write_bps_device:
  1149. case BLKIO_THROTL_read_iops_device:
  1150. case BLKIO_THROTL_write_iops_device:
  1151. blkio_read_policy_node_files(cft, blkcg, m);
  1152. return 0;
  1153. default:
  1154. BUG();
  1155. }
  1156. break;
  1157. default:
  1158. BUG();
  1159. }
  1160. return 0;
  1161. }
  1162. static int blkio_read_blkg_stats(struct blkio_cgroup *blkcg,
  1163. struct cftype *cft, struct cgroup_map_cb *cb,
  1164. enum stat_type type, bool show_total, bool pcpu)
  1165. {
  1166. struct blkio_group *blkg;
  1167. struct hlist_node *n;
  1168. uint64_t cgroup_total = 0;
  1169. rcu_read_lock();
  1170. hlist_for_each_entry_rcu(blkg, n, &blkcg->blkg_list, blkcg_node) {
  1171. if (blkg->dev) {
  1172. if (!cftype_blkg_same_policy(cft, blkg))
  1173. continue;
  1174. if (pcpu)
  1175. cgroup_total += blkio_get_stat_cpu(blkg, cb,
  1176. blkg->dev, type);
  1177. else {
  1178. spin_lock_irq(&blkg->stats_lock);
  1179. cgroup_total += blkio_get_stat(blkg, cb,
  1180. blkg->dev, type);
  1181. spin_unlock_irq(&blkg->stats_lock);
  1182. }
  1183. }
  1184. }
  1185. if (show_total)
  1186. cb->fill(cb, "Total", cgroup_total);
  1187. rcu_read_unlock();
  1188. return 0;
  1189. }
  1190. /* All map kind of cgroup file get serviced by this function */
  1191. static int blkiocg_file_read_map(struct cgroup *cgrp, struct cftype *cft,
  1192. struct cgroup_map_cb *cb)
  1193. {
  1194. struct blkio_cgroup *blkcg;
  1195. enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
  1196. int name = BLKIOFILE_ATTR(cft->private);
  1197. blkcg = cgroup_to_blkio_cgroup(cgrp);
  1198. switch(plid) {
  1199. case BLKIO_POLICY_PROP:
  1200. switch(name) {
  1201. case BLKIO_PROP_time:
  1202. return blkio_read_blkg_stats(blkcg, cft, cb,
  1203. BLKIO_STAT_TIME, 0, 0);
  1204. case BLKIO_PROP_sectors:
  1205. return blkio_read_blkg_stats(blkcg, cft, cb,
  1206. BLKIO_STAT_CPU_SECTORS, 0, 1);
  1207. case BLKIO_PROP_io_service_bytes:
  1208. return blkio_read_blkg_stats(blkcg, cft, cb,
  1209. BLKIO_STAT_CPU_SERVICE_BYTES, 1, 1);
  1210. case BLKIO_PROP_io_serviced:
  1211. return blkio_read_blkg_stats(blkcg, cft, cb,
  1212. BLKIO_STAT_CPU_SERVICED, 1, 1);
  1213. case BLKIO_PROP_io_service_time:
  1214. return blkio_read_blkg_stats(blkcg, cft, cb,
  1215. BLKIO_STAT_SERVICE_TIME, 1, 0);
  1216. case BLKIO_PROP_io_wait_time:
  1217. return blkio_read_blkg_stats(blkcg, cft, cb,
  1218. BLKIO_STAT_WAIT_TIME, 1, 0);
  1219. case BLKIO_PROP_io_merged:
  1220. return blkio_read_blkg_stats(blkcg, cft, cb,
  1221. BLKIO_STAT_CPU_MERGED, 1, 1);
  1222. case BLKIO_PROP_io_queued:
  1223. return blkio_read_blkg_stats(blkcg, cft, cb,
  1224. BLKIO_STAT_QUEUED, 1, 0);
  1225. #ifdef CONFIG_DEBUG_BLK_CGROUP
  1226. case BLKIO_PROP_unaccounted_time:
  1227. return blkio_read_blkg_stats(blkcg, cft, cb,
  1228. BLKIO_STAT_UNACCOUNTED_TIME, 0, 0);
  1229. case BLKIO_PROP_dequeue:
  1230. return blkio_read_blkg_stats(blkcg, cft, cb,
  1231. BLKIO_STAT_DEQUEUE, 0, 0);
  1232. case BLKIO_PROP_avg_queue_size:
  1233. return blkio_read_blkg_stats(blkcg, cft, cb,
  1234. BLKIO_STAT_AVG_QUEUE_SIZE, 0, 0);
  1235. case BLKIO_PROP_group_wait_time:
  1236. return blkio_read_blkg_stats(blkcg, cft, cb,
  1237. BLKIO_STAT_GROUP_WAIT_TIME, 0, 0);
  1238. case BLKIO_PROP_idle_time:
  1239. return blkio_read_blkg_stats(blkcg, cft, cb,
  1240. BLKIO_STAT_IDLE_TIME, 0, 0);
  1241. case BLKIO_PROP_empty_time:
  1242. return blkio_read_blkg_stats(blkcg, cft, cb,
  1243. BLKIO_STAT_EMPTY_TIME, 0, 0);
  1244. #endif
  1245. default:
  1246. BUG();
  1247. }
  1248. break;
  1249. case BLKIO_POLICY_THROTL:
  1250. switch(name){
  1251. case BLKIO_THROTL_io_service_bytes:
  1252. return blkio_read_blkg_stats(blkcg, cft, cb,
  1253. BLKIO_STAT_CPU_SERVICE_BYTES, 1, 1);
  1254. case BLKIO_THROTL_io_serviced:
  1255. return blkio_read_blkg_stats(blkcg, cft, cb,
  1256. BLKIO_STAT_CPU_SERVICED, 1, 1);
  1257. default:
  1258. BUG();
  1259. }
  1260. break;
  1261. default:
  1262. BUG();
  1263. }
  1264. return 0;
  1265. }
  1266. static int blkio_weight_write(struct blkio_cgroup *blkcg, u64 val)
  1267. {
  1268. struct blkio_group *blkg;
  1269. struct hlist_node *n;
  1270. struct blkio_policy_node *pn;
  1271. if (val < BLKIO_WEIGHT_MIN || val > BLKIO_WEIGHT_MAX)
  1272. return -EINVAL;
  1273. spin_lock(&blkio_list_lock);
  1274. spin_lock_irq(&blkcg->lock);
  1275. blkcg->weight = (unsigned int)val;
  1276. hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
  1277. pn = blkio_policy_search_node(blkcg, blkg->dev,
  1278. BLKIO_POLICY_PROP, BLKIO_PROP_weight_device);
  1279. if (pn)
  1280. continue;
  1281. blkio_update_group_weight(blkg, blkcg->weight);
  1282. }
  1283. spin_unlock_irq(&blkcg->lock);
  1284. spin_unlock(&blkio_list_lock);
  1285. return 0;
  1286. }
  1287. static u64 blkiocg_file_read_u64 (struct cgroup *cgrp, struct cftype *cft) {
  1288. struct blkio_cgroup *blkcg;
  1289. enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
  1290. int name = BLKIOFILE_ATTR(cft->private);
  1291. blkcg = cgroup_to_blkio_cgroup(cgrp);
  1292. switch(plid) {
  1293. case BLKIO_POLICY_PROP:
  1294. switch(name) {
  1295. case BLKIO_PROP_weight:
  1296. return (u64)blkcg->weight;
  1297. }
  1298. break;
  1299. default:
  1300. BUG();
  1301. }
  1302. return 0;
  1303. }
  1304. static int
  1305. blkiocg_file_write_u64(struct cgroup *cgrp, struct cftype *cft, u64 val)
  1306. {
  1307. struct blkio_cgroup *blkcg;
  1308. enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
  1309. int name = BLKIOFILE_ATTR(cft->private);
  1310. blkcg = cgroup_to_blkio_cgroup(cgrp);
  1311. switch(plid) {
  1312. case BLKIO_POLICY_PROP:
  1313. switch(name) {
  1314. case BLKIO_PROP_weight:
  1315. return blkio_weight_write(blkcg, val);
  1316. }
  1317. break;
  1318. default:
  1319. BUG();
  1320. }
  1321. return 0;
  1322. }
  1323. struct cftype blkio_files[] = {
  1324. {
  1325. .name = "weight_device",
  1326. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1327. BLKIO_PROP_weight_device),
  1328. .read_seq_string = blkiocg_file_read,
  1329. .write_string = blkiocg_file_write,
  1330. .max_write_len = 256,
  1331. },
  1332. {
  1333. .name = "weight",
  1334. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1335. BLKIO_PROP_weight),
  1336. .read_u64 = blkiocg_file_read_u64,
  1337. .write_u64 = blkiocg_file_write_u64,
  1338. },
  1339. {
  1340. .name = "time",
  1341. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1342. BLKIO_PROP_time),
  1343. .read_map = blkiocg_file_read_map,
  1344. },
  1345. {
  1346. .name = "sectors",
  1347. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1348. BLKIO_PROP_sectors),
  1349. .read_map = blkiocg_file_read_map,
  1350. },
  1351. {
  1352. .name = "io_service_bytes",
  1353. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1354. BLKIO_PROP_io_service_bytes),
  1355. .read_map = blkiocg_file_read_map,
  1356. },
  1357. {
  1358. .name = "io_serviced",
  1359. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1360. BLKIO_PROP_io_serviced),
  1361. .read_map = blkiocg_file_read_map,
  1362. },
  1363. {
  1364. .name = "io_service_time",
  1365. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1366. BLKIO_PROP_io_service_time),
  1367. .read_map = blkiocg_file_read_map,
  1368. },
  1369. {
  1370. .name = "io_wait_time",
  1371. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1372. BLKIO_PROP_io_wait_time),
  1373. .read_map = blkiocg_file_read_map,
  1374. },
  1375. {
  1376. .name = "io_merged",
  1377. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1378. BLKIO_PROP_io_merged),
  1379. .read_map = blkiocg_file_read_map,
  1380. },
  1381. {
  1382. .name = "io_queued",
  1383. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1384. BLKIO_PROP_io_queued),
  1385. .read_map = blkiocg_file_read_map,
  1386. },
  1387. {
  1388. .name = "reset_stats",
  1389. .write_u64 = blkiocg_reset_stats,
  1390. },
  1391. #ifdef CONFIG_BLK_DEV_THROTTLING
  1392. {
  1393. .name = "throttle.read_bps_device",
  1394. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
  1395. BLKIO_THROTL_read_bps_device),
  1396. .read_seq_string = blkiocg_file_read,
  1397. .write_string = blkiocg_file_write,
  1398. .max_write_len = 256,
  1399. },
  1400. {
  1401. .name = "throttle.write_bps_device",
  1402. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
  1403. BLKIO_THROTL_write_bps_device),
  1404. .read_seq_string = blkiocg_file_read,
  1405. .write_string = blkiocg_file_write,
  1406. .max_write_len = 256,
  1407. },
  1408. {
  1409. .name = "throttle.read_iops_device",
  1410. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
  1411. BLKIO_THROTL_read_iops_device),
  1412. .read_seq_string = blkiocg_file_read,
  1413. .write_string = blkiocg_file_write,
  1414. .max_write_len = 256,
  1415. },
  1416. {
  1417. .name = "throttle.write_iops_device",
  1418. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
  1419. BLKIO_THROTL_write_iops_device),
  1420. .read_seq_string = blkiocg_file_read,
  1421. .write_string = blkiocg_file_write,
  1422. .max_write_len = 256,
  1423. },
  1424. {
  1425. .name = "throttle.io_service_bytes",
  1426. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
  1427. BLKIO_THROTL_io_service_bytes),
  1428. .read_map = blkiocg_file_read_map,
  1429. },
  1430. {
  1431. .name = "throttle.io_serviced",
  1432. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
  1433. BLKIO_THROTL_io_serviced),
  1434. .read_map = blkiocg_file_read_map,
  1435. },
  1436. #endif /* CONFIG_BLK_DEV_THROTTLING */
  1437. #ifdef CONFIG_DEBUG_BLK_CGROUP
  1438. {
  1439. .name = "avg_queue_size",
  1440. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1441. BLKIO_PROP_avg_queue_size),
  1442. .read_map = blkiocg_file_read_map,
  1443. },
  1444. {
  1445. .name = "group_wait_time",
  1446. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1447. BLKIO_PROP_group_wait_time),
  1448. .read_map = blkiocg_file_read_map,
  1449. },
  1450. {
  1451. .name = "idle_time",
  1452. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1453. BLKIO_PROP_idle_time),
  1454. .read_map = blkiocg_file_read_map,
  1455. },
  1456. {
  1457. .name = "empty_time",
  1458. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1459. BLKIO_PROP_empty_time),
  1460. .read_map = blkiocg_file_read_map,
  1461. },
  1462. {
  1463. .name = "dequeue",
  1464. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1465. BLKIO_PROP_dequeue),
  1466. .read_map = blkiocg_file_read_map,
  1467. },
  1468. {
  1469. .name = "unaccounted_time",
  1470. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1471. BLKIO_PROP_unaccounted_time),
  1472. .read_map = blkiocg_file_read_map,
  1473. },
  1474. #endif
  1475. };
  1476. static int blkiocg_populate(struct cgroup_subsys *subsys, struct cgroup *cgroup)
  1477. {
  1478. return cgroup_add_files(cgroup, subsys, blkio_files,
  1479. ARRAY_SIZE(blkio_files));
  1480. }
  1481. static void blkiocg_destroy(struct cgroup_subsys *subsys, struct cgroup *cgroup)
  1482. {
  1483. struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgroup);
  1484. unsigned long flags;
  1485. struct blkio_group *blkg;
  1486. struct request_queue *q;
  1487. struct blkio_policy_type *blkiop;
  1488. struct blkio_policy_node *pn, *pntmp;
  1489. rcu_read_lock();
  1490. do {
  1491. spin_lock_irqsave(&blkcg->lock, flags);
  1492. if (hlist_empty(&blkcg->blkg_list)) {
  1493. spin_unlock_irqrestore(&blkcg->lock, flags);
  1494. break;
  1495. }
  1496. blkg = hlist_entry(blkcg->blkg_list.first, struct blkio_group,
  1497. blkcg_node);
  1498. q = rcu_dereference(blkg->q);
  1499. __blkiocg_del_blkio_group(blkg);
  1500. spin_unlock_irqrestore(&blkcg->lock, flags);
  1501. /*
  1502. * This blkio_group is being unlinked as associated cgroup is
  1503. * going away. Let all the IO controlling policies know about
  1504. * this event.
  1505. */
  1506. spin_lock(&blkio_list_lock);
  1507. list_for_each_entry(blkiop, &blkio_list, list) {
  1508. if (blkiop->plid != blkg->plid)
  1509. continue;
  1510. blkiop->ops.blkio_unlink_group_fn(q, blkg);
  1511. }
  1512. spin_unlock(&blkio_list_lock);
  1513. } while (1);
  1514. list_for_each_entry_safe(pn, pntmp, &blkcg->policy_list, node) {
  1515. blkio_policy_delete_node(pn);
  1516. kfree(pn);
  1517. }
  1518. free_css_id(&blkio_subsys, &blkcg->css);
  1519. rcu_read_unlock();
  1520. if (blkcg != &blkio_root_cgroup)
  1521. kfree(blkcg);
  1522. }
  1523. static struct cgroup_subsys_state *
  1524. blkiocg_create(struct cgroup_subsys *subsys, struct cgroup *cgroup)
  1525. {
  1526. struct blkio_cgroup *blkcg;
  1527. struct cgroup *parent = cgroup->parent;
  1528. if (!parent) {
  1529. blkcg = &blkio_root_cgroup;
  1530. goto done;
  1531. }
  1532. blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
  1533. if (!blkcg)
  1534. return ERR_PTR(-ENOMEM);
  1535. blkcg->weight = BLKIO_WEIGHT_DEFAULT;
  1536. done:
  1537. spin_lock_init(&blkcg->lock);
  1538. INIT_HLIST_HEAD(&blkcg->blkg_list);
  1539. INIT_LIST_HEAD(&blkcg->policy_list);
  1540. return &blkcg->css;
  1541. }
  1542. /*
  1543. * We cannot support shared io contexts, as we have no mean to support
  1544. * two tasks with the same ioc in two different groups without major rework
  1545. * of the main cic data structures. For now we allow a task to change
  1546. * its cgroup only if it's the only owner of its ioc.
  1547. */
  1548. static int blkiocg_can_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
  1549. struct cgroup_taskset *tset)
  1550. {
  1551. struct task_struct *task;
  1552. struct io_context *ioc;
  1553. int ret = 0;
  1554. /* task_lock() is needed to avoid races with exit_io_context() */
  1555. cgroup_taskset_for_each(task, cgrp, tset) {
  1556. task_lock(task);
  1557. ioc = task->io_context;
  1558. if (ioc && atomic_read(&ioc->nr_tasks) > 1)
  1559. ret = -EINVAL;
  1560. task_unlock(task);
  1561. if (ret)
  1562. break;
  1563. }
  1564. return ret;
  1565. }
  1566. static void blkiocg_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
  1567. struct cgroup_taskset *tset)
  1568. {
  1569. struct task_struct *task;
  1570. struct io_context *ioc;
  1571. cgroup_taskset_for_each(task, cgrp, tset) {
  1572. /* we don't lose anything even if ioc allocation fails */
  1573. ioc = get_task_io_context(task, GFP_ATOMIC, NUMA_NO_NODE);
  1574. if (ioc) {
  1575. ioc_cgroup_changed(ioc);
  1576. put_io_context(ioc);
  1577. }
  1578. }
  1579. }
  1580. void blkio_policy_register(struct blkio_policy_type *blkiop)
  1581. {
  1582. spin_lock(&blkio_list_lock);
  1583. BUG_ON(blkio_policy[blkiop->plid]);
  1584. blkio_policy[blkiop->plid] = blkiop;
  1585. list_add_tail(&blkiop->list, &blkio_list);
  1586. spin_unlock(&blkio_list_lock);
  1587. }
  1588. EXPORT_SYMBOL_GPL(blkio_policy_register);
  1589. void blkio_policy_unregister(struct blkio_policy_type *blkiop)
  1590. {
  1591. spin_lock(&blkio_list_lock);
  1592. BUG_ON(blkio_policy[blkiop->plid] != blkiop);
  1593. blkio_policy[blkiop->plid] = NULL;
  1594. list_del_init(&blkiop->list);
  1595. spin_unlock(&blkio_list_lock);
  1596. }
  1597. EXPORT_SYMBOL_GPL(blkio_policy_unregister);