blk-cgroup.c 46 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806
  1. /*
  2. * Common Block IO controller cgroup interface
  3. *
  4. * Based on ideas and code from CFQ, CFS and BFQ:
  5. * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
  6. *
  7. * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
  8. * Paolo Valente <paolo.valente@unimore.it>
  9. *
  10. * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
  11. * Nauman Rafique <nauman@google.com>
  12. */
  13. #include <linux/ioprio.h>
  14. #include <linux/seq_file.h>
  15. #include <linux/kdev_t.h>
  16. #include <linux/module.h>
  17. #include <linux/err.h>
  18. #include <linux/blkdev.h>
  19. #include <linux/slab.h>
  20. #include <linux/genhd.h>
  21. #include <linux/delay.h>
  22. #include <linux/atomic.h>
  23. #include "blk-cgroup.h"
  24. #include "blk.h"
  25. #define MAX_KEY_LEN 100
  26. static DEFINE_SPINLOCK(blkio_list_lock);
  27. static LIST_HEAD(blkio_list);
  28. static DEFINE_MUTEX(all_q_mutex);
  29. static LIST_HEAD(all_q_list);
  30. /* List of groups pending per cpu stats allocation */
  31. static DEFINE_SPINLOCK(alloc_list_lock);
  32. static LIST_HEAD(alloc_list);
  33. static void blkio_stat_alloc_fn(struct work_struct *);
  34. static DECLARE_DELAYED_WORK(blkio_stat_alloc_work, blkio_stat_alloc_fn);
  35. struct blkio_cgroup blkio_root_cgroup = { .weight = 2*BLKIO_WEIGHT_DEFAULT };
  36. EXPORT_SYMBOL_GPL(blkio_root_cgroup);
  37. static struct blkio_policy_type *blkio_policy[BLKIO_NR_POLICIES];
  38. static struct cgroup_subsys_state *blkiocg_create(struct cgroup_subsys *,
  39. struct cgroup *);
  40. static int blkiocg_can_attach(struct cgroup_subsys *, struct cgroup *,
  41. struct cgroup_taskset *);
  42. static void blkiocg_attach(struct cgroup_subsys *, struct cgroup *,
  43. struct cgroup_taskset *);
  44. static int blkiocg_pre_destroy(struct cgroup_subsys *, struct cgroup *);
  45. static void blkiocg_destroy(struct cgroup_subsys *, struct cgroup *);
  46. static int blkiocg_populate(struct cgroup_subsys *, struct cgroup *);
  47. /* for encoding cft->private value on file */
  48. #define BLKIOFILE_PRIVATE(x, val) (((x) << 16) | (val))
  49. /* What policy owns the file, proportional or throttle */
  50. #define BLKIOFILE_POLICY(val) (((val) >> 16) & 0xffff)
  51. #define BLKIOFILE_ATTR(val) ((val) & 0xffff)
  52. struct cgroup_subsys blkio_subsys = {
  53. .name = "blkio",
  54. .create = blkiocg_create,
  55. .can_attach = blkiocg_can_attach,
  56. .attach = blkiocg_attach,
  57. .pre_destroy = blkiocg_pre_destroy,
  58. .destroy = blkiocg_destroy,
  59. .populate = blkiocg_populate,
  60. .subsys_id = blkio_subsys_id,
  61. .module = THIS_MODULE,
  62. };
  63. EXPORT_SYMBOL_GPL(blkio_subsys);
  64. struct blkio_cgroup *cgroup_to_blkio_cgroup(struct cgroup *cgroup)
  65. {
  66. return container_of(cgroup_subsys_state(cgroup, blkio_subsys_id),
  67. struct blkio_cgroup, css);
  68. }
  69. EXPORT_SYMBOL_GPL(cgroup_to_blkio_cgroup);
  70. static struct blkio_cgroup *task_blkio_cgroup(struct task_struct *tsk)
  71. {
  72. return container_of(task_subsys_state(tsk, blkio_subsys_id),
  73. struct blkio_cgroup, css);
  74. }
  75. struct blkio_cgroup *bio_blkio_cgroup(struct bio *bio)
  76. {
  77. if (bio && bio->bi_css)
  78. return container_of(bio->bi_css, struct blkio_cgroup, css);
  79. return task_blkio_cgroup(current);
  80. }
  81. EXPORT_SYMBOL_GPL(bio_blkio_cgroup);
  82. static inline void blkio_update_group_weight(struct blkio_group *blkg,
  83. int plid, unsigned int weight)
  84. {
  85. struct blkio_policy_type *blkiop;
  86. list_for_each_entry(blkiop, &blkio_list, list) {
  87. /* If this policy does not own the blkg, do not send updates */
  88. if (blkiop->plid != plid)
  89. continue;
  90. if (blkiop->ops.blkio_update_group_weight_fn)
  91. blkiop->ops.blkio_update_group_weight_fn(blkg->q,
  92. blkg, weight);
  93. }
  94. }
  95. static inline void blkio_update_group_bps(struct blkio_group *blkg, int plid,
  96. u64 bps, int fileid)
  97. {
  98. struct blkio_policy_type *blkiop;
  99. list_for_each_entry(blkiop, &blkio_list, list) {
  100. /* If this policy does not own the blkg, do not send updates */
  101. if (blkiop->plid != plid)
  102. continue;
  103. if (fileid == BLKIO_THROTL_read_bps_device
  104. && blkiop->ops.blkio_update_group_read_bps_fn)
  105. blkiop->ops.blkio_update_group_read_bps_fn(blkg->q,
  106. blkg, bps);
  107. if (fileid == BLKIO_THROTL_write_bps_device
  108. && blkiop->ops.blkio_update_group_write_bps_fn)
  109. blkiop->ops.blkio_update_group_write_bps_fn(blkg->q,
  110. blkg, bps);
  111. }
  112. }
  113. static inline void blkio_update_group_iops(struct blkio_group *blkg,
  114. int plid, unsigned int iops,
  115. int fileid)
  116. {
  117. struct blkio_policy_type *blkiop;
  118. list_for_each_entry(blkiop, &blkio_list, list) {
  119. /* If this policy does not own the blkg, do not send updates */
  120. if (blkiop->plid != plid)
  121. continue;
  122. if (fileid == BLKIO_THROTL_read_iops_device
  123. && blkiop->ops.blkio_update_group_read_iops_fn)
  124. blkiop->ops.blkio_update_group_read_iops_fn(blkg->q,
  125. blkg, iops);
  126. if (fileid == BLKIO_THROTL_write_iops_device
  127. && blkiop->ops.blkio_update_group_write_iops_fn)
  128. blkiop->ops.blkio_update_group_write_iops_fn(blkg->q,
  129. blkg,iops);
  130. }
  131. }
  132. /*
  133. * Add to the appropriate stat variable depending on the request type.
  134. * This should be called with queue_lock held.
  135. */
  136. static void blkio_add_stat(uint64_t *stat, uint64_t add, bool direction,
  137. bool sync)
  138. {
  139. if (direction)
  140. stat[BLKIO_STAT_WRITE] += add;
  141. else
  142. stat[BLKIO_STAT_READ] += add;
  143. if (sync)
  144. stat[BLKIO_STAT_SYNC] += add;
  145. else
  146. stat[BLKIO_STAT_ASYNC] += add;
  147. }
  148. /*
  149. * Decrements the appropriate stat variable if non-zero depending on the
  150. * request type. Panics on value being zero.
  151. * This should be called with the queue_lock held.
  152. */
  153. static void blkio_check_and_dec_stat(uint64_t *stat, bool direction, bool sync)
  154. {
  155. if (direction) {
  156. BUG_ON(stat[BLKIO_STAT_WRITE] == 0);
  157. stat[BLKIO_STAT_WRITE]--;
  158. } else {
  159. BUG_ON(stat[BLKIO_STAT_READ] == 0);
  160. stat[BLKIO_STAT_READ]--;
  161. }
  162. if (sync) {
  163. BUG_ON(stat[BLKIO_STAT_SYNC] == 0);
  164. stat[BLKIO_STAT_SYNC]--;
  165. } else {
  166. BUG_ON(stat[BLKIO_STAT_ASYNC] == 0);
  167. stat[BLKIO_STAT_ASYNC]--;
  168. }
  169. }
  170. #ifdef CONFIG_DEBUG_BLK_CGROUP
  171. /* This should be called with the queue_lock held. */
  172. static void blkio_set_start_group_wait_time(struct blkio_group *blkg,
  173. struct blkio_policy_type *pol,
  174. struct blkio_group *curr_blkg)
  175. {
  176. struct blkg_policy_data *pd = blkg->pd[pol->plid];
  177. if (blkio_blkg_waiting(&pd->stats))
  178. return;
  179. if (blkg == curr_blkg)
  180. return;
  181. pd->stats.start_group_wait_time = sched_clock();
  182. blkio_mark_blkg_waiting(&pd->stats);
  183. }
  184. /* This should be called with the queue_lock held. */
  185. static void blkio_update_group_wait_time(struct blkio_group_stats *stats)
  186. {
  187. unsigned long long now;
  188. if (!blkio_blkg_waiting(stats))
  189. return;
  190. now = sched_clock();
  191. if (time_after64(now, stats->start_group_wait_time))
  192. stats->group_wait_time += now - stats->start_group_wait_time;
  193. blkio_clear_blkg_waiting(stats);
  194. }
  195. /* This should be called with the queue_lock held. */
  196. static void blkio_end_empty_time(struct blkio_group_stats *stats)
  197. {
  198. unsigned long long now;
  199. if (!blkio_blkg_empty(stats))
  200. return;
  201. now = sched_clock();
  202. if (time_after64(now, stats->start_empty_time))
  203. stats->empty_time += now - stats->start_empty_time;
  204. blkio_clear_blkg_empty(stats);
  205. }
  206. void blkiocg_update_set_idle_time_stats(struct blkio_group *blkg,
  207. struct blkio_policy_type *pol)
  208. {
  209. struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
  210. lockdep_assert_held(blkg->q->queue_lock);
  211. BUG_ON(blkio_blkg_idling(stats));
  212. stats->start_idle_time = sched_clock();
  213. blkio_mark_blkg_idling(stats);
  214. }
  215. EXPORT_SYMBOL_GPL(blkiocg_update_set_idle_time_stats);
  216. void blkiocg_update_idle_time_stats(struct blkio_group *blkg,
  217. struct blkio_policy_type *pol)
  218. {
  219. struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
  220. lockdep_assert_held(blkg->q->queue_lock);
  221. if (blkio_blkg_idling(stats)) {
  222. unsigned long long now = sched_clock();
  223. if (time_after64(now, stats->start_idle_time)) {
  224. u64_stats_update_begin(&stats->syncp);
  225. stats->idle_time += now - stats->start_idle_time;
  226. u64_stats_update_end(&stats->syncp);
  227. }
  228. blkio_clear_blkg_idling(stats);
  229. }
  230. }
  231. EXPORT_SYMBOL_GPL(blkiocg_update_idle_time_stats);
  232. void blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg,
  233. struct blkio_policy_type *pol)
  234. {
  235. struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
  236. lockdep_assert_held(blkg->q->queue_lock);
  237. u64_stats_update_begin(&stats->syncp);
  238. stats->avg_queue_size_sum +=
  239. stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_READ] +
  240. stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_WRITE];
  241. stats->avg_queue_size_samples++;
  242. blkio_update_group_wait_time(stats);
  243. u64_stats_update_end(&stats->syncp);
  244. }
  245. EXPORT_SYMBOL_GPL(blkiocg_update_avg_queue_size_stats);
  246. void blkiocg_set_start_empty_time(struct blkio_group *blkg,
  247. struct blkio_policy_type *pol)
  248. {
  249. struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
  250. lockdep_assert_held(blkg->q->queue_lock);
  251. if (stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_READ] ||
  252. stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_WRITE])
  253. return;
  254. /*
  255. * group is already marked empty. This can happen if cfqq got new
  256. * request in parent group and moved to this group while being added
  257. * to service tree. Just ignore the event and move on.
  258. */
  259. if (blkio_blkg_empty(stats))
  260. return;
  261. stats->start_empty_time = sched_clock();
  262. blkio_mark_blkg_empty(stats);
  263. }
  264. EXPORT_SYMBOL_GPL(blkiocg_set_start_empty_time);
  265. void blkiocg_update_dequeue_stats(struct blkio_group *blkg,
  266. struct blkio_policy_type *pol,
  267. unsigned long dequeue)
  268. {
  269. struct blkg_policy_data *pd = blkg->pd[pol->plid];
  270. lockdep_assert_held(blkg->q->queue_lock);
  271. pd->stats.dequeue += dequeue;
  272. }
  273. EXPORT_SYMBOL_GPL(blkiocg_update_dequeue_stats);
  274. #else
  275. static inline void blkio_set_start_group_wait_time(struct blkio_group *blkg,
  276. struct blkio_policy_type *pol,
  277. struct blkio_group *curr_blkg) { }
  278. static inline void blkio_end_empty_time(struct blkio_group_stats *stats) { }
  279. #endif
  280. void blkiocg_update_io_add_stats(struct blkio_group *blkg,
  281. struct blkio_policy_type *pol,
  282. struct blkio_group *curr_blkg, bool direction,
  283. bool sync)
  284. {
  285. struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
  286. lockdep_assert_held(blkg->q->queue_lock);
  287. u64_stats_update_begin(&stats->syncp);
  288. blkio_add_stat(stats->stat_arr[BLKIO_STAT_QUEUED], 1, direction, sync);
  289. blkio_end_empty_time(stats);
  290. u64_stats_update_end(&stats->syncp);
  291. blkio_set_start_group_wait_time(blkg, pol, curr_blkg);
  292. }
  293. EXPORT_SYMBOL_GPL(blkiocg_update_io_add_stats);
  294. void blkiocg_update_io_remove_stats(struct blkio_group *blkg,
  295. struct blkio_policy_type *pol,
  296. bool direction, bool sync)
  297. {
  298. struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
  299. lockdep_assert_held(blkg->q->queue_lock);
  300. u64_stats_update_begin(&stats->syncp);
  301. blkio_check_and_dec_stat(stats->stat_arr[BLKIO_STAT_QUEUED], direction,
  302. sync);
  303. u64_stats_update_end(&stats->syncp);
  304. }
  305. EXPORT_SYMBOL_GPL(blkiocg_update_io_remove_stats);
  306. void blkiocg_update_timeslice_used(struct blkio_group *blkg,
  307. struct blkio_policy_type *pol,
  308. unsigned long time,
  309. unsigned long unaccounted_time)
  310. {
  311. struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
  312. lockdep_assert_held(blkg->q->queue_lock);
  313. u64_stats_update_begin(&stats->syncp);
  314. stats->time += time;
  315. #ifdef CONFIG_DEBUG_BLK_CGROUP
  316. stats->unaccounted_time += unaccounted_time;
  317. #endif
  318. u64_stats_update_end(&stats->syncp);
  319. }
  320. EXPORT_SYMBOL_GPL(blkiocg_update_timeslice_used);
  321. /*
  322. * should be called under rcu read lock or queue lock to make sure blkg pointer
  323. * is valid.
  324. */
  325. void blkiocg_update_dispatch_stats(struct blkio_group *blkg,
  326. struct blkio_policy_type *pol,
  327. uint64_t bytes, bool direction, bool sync)
  328. {
  329. struct blkg_policy_data *pd = blkg->pd[pol->plid];
  330. struct blkio_group_stats_cpu *stats_cpu;
  331. unsigned long flags;
  332. /* If per cpu stats are not allocated yet, don't do any accounting. */
  333. if (pd->stats_cpu == NULL)
  334. return;
  335. /*
  336. * Disabling interrupts to provide mutual exclusion between two
  337. * writes on same cpu. It probably is not needed for 64bit. Not
  338. * optimizing that case yet.
  339. */
  340. local_irq_save(flags);
  341. stats_cpu = this_cpu_ptr(pd->stats_cpu);
  342. u64_stats_update_begin(&stats_cpu->syncp);
  343. stats_cpu->sectors += bytes >> 9;
  344. blkio_add_stat(stats_cpu->stat_arr_cpu[BLKIO_STAT_CPU_SERVICED],
  345. 1, direction, sync);
  346. blkio_add_stat(stats_cpu->stat_arr_cpu[BLKIO_STAT_CPU_SERVICE_BYTES],
  347. bytes, direction, sync);
  348. u64_stats_update_end(&stats_cpu->syncp);
  349. local_irq_restore(flags);
  350. }
  351. EXPORT_SYMBOL_GPL(blkiocg_update_dispatch_stats);
  352. void blkiocg_update_completion_stats(struct blkio_group *blkg,
  353. struct blkio_policy_type *pol,
  354. uint64_t start_time,
  355. uint64_t io_start_time, bool direction,
  356. bool sync)
  357. {
  358. struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
  359. unsigned long long now = sched_clock();
  360. lockdep_assert_held(blkg->q->queue_lock);
  361. u64_stats_update_begin(&stats->syncp);
  362. if (time_after64(now, io_start_time))
  363. blkio_add_stat(stats->stat_arr[BLKIO_STAT_SERVICE_TIME],
  364. now - io_start_time, direction, sync);
  365. if (time_after64(io_start_time, start_time))
  366. blkio_add_stat(stats->stat_arr[BLKIO_STAT_WAIT_TIME],
  367. io_start_time - start_time, direction, sync);
  368. u64_stats_update_end(&stats->syncp);
  369. }
  370. EXPORT_SYMBOL_GPL(blkiocg_update_completion_stats);
  371. /* Merged stats are per cpu. */
  372. void blkiocg_update_io_merged_stats(struct blkio_group *blkg,
  373. struct blkio_policy_type *pol,
  374. bool direction, bool sync)
  375. {
  376. struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
  377. lockdep_assert_held(blkg->q->queue_lock);
  378. u64_stats_update_begin(&stats->syncp);
  379. blkio_add_stat(stats->stat_arr[BLKIO_STAT_MERGED], 1, direction, sync);
  380. u64_stats_update_end(&stats->syncp);
  381. }
  382. EXPORT_SYMBOL_GPL(blkiocg_update_io_merged_stats);
  383. /*
  384. * Worker for allocating per cpu stat for blk groups. This is scheduled on
  385. * the system_nrt_wq once there are some groups on the alloc_list waiting
  386. * for allocation.
  387. */
  388. static void blkio_stat_alloc_fn(struct work_struct *work)
  389. {
  390. static void *pcpu_stats[BLKIO_NR_POLICIES];
  391. struct delayed_work *dwork = to_delayed_work(work);
  392. struct blkio_group *blkg;
  393. int i;
  394. bool empty = false;
  395. alloc_stats:
  396. for (i = 0; i < BLKIO_NR_POLICIES; i++) {
  397. if (pcpu_stats[i] != NULL)
  398. continue;
  399. pcpu_stats[i] = alloc_percpu(struct blkio_group_stats_cpu);
  400. /* Allocation failed. Try again after some time. */
  401. if (pcpu_stats[i] == NULL) {
  402. queue_delayed_work(system_nrt_wq, dwork,
  403. msecs_to_jiffies(10));
  404. return;
  405. }
  406. }
  407. spin_lock_irq(&blkio_list_lock);
  408. spin_lock(&alloc_list_lock);
  409. /* cgroup got deleted or queue exited. */
  410. if (!list_empty(&alloc_list)) {
  411. blkg = list_first_entry(&alloc_list, struct blkio_group,
  412. alloc_node);
  413. for (i = 0; i < BLKIO_NR_POLICIES; i++) {
  414. struct blkg_policy_data *pd = blkg->pd[i];
  415. if (blkio_policy[i] && pd && !pd->stats_cpu)
  416. swap(pd->stats_cpu, pcpu_stats[i]);
  417. }
  418. list_del_init(&blkg->alloc_node);
  419. }
  420. empty = list_empty(&alloc_list);
  421. spin_unlock(&alloc_list_lock);
  422. spin_unlock_irq(&blkio_list_lock);
  423. if (!empty)
  424. goto alloc_stats;
  425. }
  426. /**
  427. * blkg_free - free a blkg
  428. * @blkg: blkg to free
  429. *
  430. * Free @blkg which may be partially allocated.
  431. */
  432. static void blkg_free(struct blkio_group *blkg)
  433. {
  434. int i;
  435. if (!blkg)
  436. return;
  437. for (i = 0; i < BLKIO_NR_POLICIES; i++) {
  438. struct blkg_policy_data *pd = blkg->pd[i];
  439. if (pd) {
  440. free_percpu(pd->stats_cpu);
  441. kfree(pd);
  442. }
  443. }
  444. kfree(blkg);
  445. }
  446. /**
  447. * blkg_alloc - allocate a blkg
  448. * @blkcg: block cgroup the new blkg is associated with
  449. * @q: request_queue the new blkg is associated with
  450. *
  451. * Allocate a new blkg assocating @blkcg and @q.
  452. */
  453. static struct blkio_group *blkg_alloc(struct blkio_cgroup *blkcg,
  454. struct request_queue *q)
  455. {
  456. struct blkio_group *blkg;
  457. int i;
  458. /* alloc and init base part */
  459. blkg = kzalloc_node(sizeof(*blkg), GFP_ATOMIC, q->node);
  460. if (!blkg)
  461. return NULL;
  462. blkg->q = q;
  463. INIT_LIST_HEAD(&blkg->q_node);
  464. INIT_LIST_HEAD(&blkg->alloc_node);
  465. blkg->blkcg = blkcg;
  466. blkg->refcnt = 1;
  467. cgroup_path(blkcg->css.cgroup, blkg->path, sizeof(blkg->path));
  468. for (i = 0; i < BLKIO_NR_POLICIES; i++) {
  469. struct blkio_policy_type *pol = blkio_policy[i];
  470. struct blkg_policy_data *pd;
  471. if (!pol)
  472. continue;
  473. /* alloc per-policy data and attach it to blkg */
  474. pd = kzalloc_node(sizeof(*pd) + pol->pdata_size, GFP_ATOMIC,
  475. q->node);
  476. if (!pd) {
  477. blkg_free(blkg);
  478. return NULL;
  479. }
  480. blkg->pd[i] = pd;
  481. pd->blkg = blkg;
  482. }
  483. /* invoke per-policy init */
  484. for (i = 0; i < BLKIO_NR_POLICIES; i++) {
  485. struct blkio_policy_type *pol = blkio_policy[i];
  486. if (pol)
  487. pol->ops.blkio_init_group_fn(blkg);
  488. }
  489. return blkg;
  490. }
  491. struct blkio_group *blkg_lookup_create(struct blkio_cgroup *blkcg,
  492. struct request_queue *q,
  493. enum blkio_policy_id plid,
  494. bool for_root)
  495. __releases(q->queue_lock) __acquires(q->queue_lock)
  496. {
  497. struct blkio_group *blkg;
  498. WARN_ON_ONCE(!rcu_read_lock_held());
  499. lockdep_assert_held(q->queue_lock);
  500. /*
  501. * This could be the first entry point of blkcg implementation and
  502. * we shouldn't allow anything to go through for a bypassing queue.
  503. * The following can be removed if blkg lookup is guaranteed to
  504. * fail on a bypassing queue.
  505. */
  506. if (unlikely(blk_queue_bypass(q)) && !for_root)
  507. return ERR_PTR(blk_queue_dead(q) ? -EINVAL : -EBUSY);
  508. blkg = blkg_lookup(blkcg, q);
  509. if (blkg)
  510. return blkg;
  511. /* blkg holds a reference to blkcg */
  512. if (!css_tryget(&blkcg->css))
  513. return ERR_PTR(-EINVAL);
  514. /*
  515. * Allocate and initialize.
  516. */
  517. blkg = blkg_alloc(blkcg, q);
  518. /* did alloc fail? */
  519. if (unlikely(!blkg)) {
  520. blkg = ERR_PTR(-ENOMEM);
  521. goto out;
  522. }
  523. /* insert */
  524. spin_lock(&blkcg->lock);
  525. hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
  526. list_add(&blkg->q_node, &q->blkg_list);
  527. spin_unlock(&blkcg->lock);
  528. spin_lock(&alloc_list_lock);
  529. list_add(&blkg->alloc_node, &alloc_list);
  530. /* Queue per cpu stat allocation from worker thread. */
  531. queue_delayed_work(system_nrt_wq, &blkio_stat_alloc_work, 0);
  532. spin_unlock(&alloc_list_lock);
  533. out:
  534. return blkg;
  535. }
  536. EXPORT_SYMBOL_GPL(blkg_lookup_create);
  537. /* called under rcu_read_lock(). */
  538. struct blkio_group *blkg_lookup(struct blkio_cgroup *blkcg,
  539. struct request_queue *q)
  540. {
  541. struct blkio_group *blkg;
  542. struct hlist_node *n;
  543. hlist_for_each_entry_rcu(blkg, n, &blkcg->blkg_list, blkcg_node)
  544. if (blkg->q == q)
  545. return blkg;
  546. return NULL;
  547. }
  548. EXPORT_SYMBOL_GPL(blkg_lookup);
  549. static void blkg_destroy(struct blkio_group *blkg)
  550. {
  551. struct request_queue *q = blkg->q;
  552. struct blkio_cgroup *blkcg = blkg->blkcg;
  553. lockdep_assert_held(q->queue_lock);
  554. lockdep_assert_held(&blkcg->lock);
  555. /* Something wrong if we are trying to remove same group twice */
  556. WARN_ON_ONCE(list_empty(&blkg->q_node));
  557. WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node));
  558. list_del_init(&blkg->q_node);
  559. hlist_del_init_rcu(&blkg->blkcg_node);
  560. spin_lock(&alloc_list_lock);
  561. list_del_init(&blkg->alloc_node);
  562. spin_unlock(&alloc_list_lock);
  563. /*
  564. * Put the reference taken at the time of creation so that when all
  565. * queues are gone, group can be destroyed.
  566. */
  567. blkg_put(blkg);
  568. }
  569. /*
  570. * XXX: This updates blkg policy data in-place for root blkg, which is
  571. * necessary across elevator switch and policy registration as root blkgs
  572. * aren't shot down. This broken and racy implementation is temporary.
  573. * Eventually, blkg shoot down will be replaced by proper in-place update.
  574. */
  575. void update_root_blkg_pd(struct request_queue *q, enum blkio_policy_id plid)
  576. {
  577. struct blkio_policy_type *pol = blkio_policy[plid];
  578. struct blkio_group *blkg = blkg_lookup(&blkio_root_cgroup, q);
  579. struct blkg_policy_data *pd;
  580. if (!blkg)
  581. return;
  582. kfree(blkg->pd[plid]);
  583. blkg->pd[plid] = NULL;
  584. if (!pol)
  585. return;
  586. pd = kzalloc(sizeof(*pd) + pol->pdata_size, GFP_KERNEL);
  587. WARN_ON_ONCE(!pd);
  588. pd->stats_cpu = alloc_percpu(struct blkio_group_stats_cpu);
  589. WARN_ON_ONCE(!pd->stats_cpu);
  590. blkg->pd[plid] = pd;
  591. pd->blkg = blkg;
  592. pol->ops.blkio_init_group_fn(blkg);
  593. }
  594. EXPORT_SYMBOL_GPL(update_root_blkg_pd);
  595. /**
  596. * blkg_destroy_all - destroy all blkgs associated with a request_queue
  597. * @q: request_queue of interest
  598. * @destroy_root: whether to destroy root blkg or not
  599. *
  600. * Destroy blkgs associated with @q. If @destroy_root is %true, all are
  601. * destroyed; otherwise, root blkg is left alone.
  602. */
  603. void blkg_destroy_all(struct request_queue *q, bool destroy_root)
  604. {
  605. struct blkio_group *blkg, *n;
  606. spin_lock_irq(q->queue_lock);
  607. list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) {
  608. struct blkio_cgroup *blkcg = blkg->blkcg;
  609. /* skip root? */
  610. if (!destroy_root && blkg->blkcg == &blkio_root_cgroup)
  611. continue;
  612. spin_lock(&blkcg->lock);
  613. blkg_destroy(blkg);
  614. spin_unlock(&blkcg->lock);
  615. }
  616. spin_unlock_irq(q->queue_lock);
  617. }
  618. EXPORT_SYMBOL_GPL(blkg_destroy_all);
  619. static void blkg_rcu_free(struct rcu_head *rcu_head)
  620. {
  621. blkg_free(container_of(rcu_head, struct blkio_group, rcu_head));
  622. }
  623. void __blkg_release(struct blkio_group *blkg)
  624. {
  625. /* release the extra blkcg reference this blkg has been holding */
  626. css_put(&blkg->blkcg->css);
  627. /*
  628. * A group is freed in rcu manner. But having an rcu lock does not
  629. * mean that one can access all the fields of blkg and assume these
  630. * are valid. For example, don't try to follow throtl_data and
  631. * request queue links.
  632. *
  633. * Having a reference to blkg under an rcu allows acess to only
  634. * values local to groups like group stats and group rate limits
  635. */
  636. call_rcu(&blkg->rcu_head, blkg_rcu_free);
  637. }
  638. EXPORT_SYMBOL_GPL(__blkg_release);
  639. static void blkio_reset_stats_cpu(struct blkio_group *blkg, int plid)
  640. {
  641. struct blkg_policy_data *pd = blkg->pd[plid];
  642. int cpu;
  643. if (pd->stats_cpu == NULL)
  644. return;
  645. for_each_possible_cpu(cpu) {
  646. struct blkio_group_stats_cpu *sc =
  647. per_cpu_ptr(pd->stats_cpu, cpu);
  648. sc->sectors = 0;
  649. memset(sc->stat_arr_cpu, 0, sizeof(sc->stat_arr_cpu));
  650. }
  651. }
  652. static int
  653. blkiocg_reset_stats(struct cgroup *cgroup, struct cftype *cftype, u64 val)
  654. {
  655. struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgroup);
  656. struct blkio_group *blkg;
  657. struct hlist_node *n;
  658. int i;
  659. spin_lock(&blkio_list_lock);
  660. spin_lock_irq(&blkcg->lock);
  661. /*
  662. * Note that stat reset is racy - it doesn't synchronize against
  663. * stat updates. This is a debug feature which shouldn't exist
  664. * anyway. If you get hit by a race, retry.
  665. */
  666. hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
  667. struct blkio_policy_type *pol;
  668. list_for_each_entry(pol, &blkio_list, list) {
  669. struct blkg_policy_data *pd = blkg->pd[pol->plid];
  670. struct blkio_group_stats *stats = &pd->stats;
  671. /* queued stats shouldn't be cleared */
  672. for (i = 0; i < ARRAY_SIZE(stats->stat_arr); i++)
  673. if (i != BLKIO_STAT_QUEUED)
  674. memset(stats->stat_arr[i], 0,
  675. sizeof(stats->stat_arr[i]));
  676. stats->time = 0;
  677. #ifdef CONFIG_DEBUG_BLK_CGROUP
  678. memset((void *)stats + BLKG_STATS_DEBUG_CLEAR_START, 0,
  679. BLKG_STATS_DEBUG_CLEAR_SIZE);
  680. #endif
  681. blkio_reset_stats_cpu(blkg, pol->plid);
  682. }
  683. }
  684. spin_unlock_irq(&blkcg->lock);
  685. spin_unlock(&blkio_list_lock);
  686. return 0;
  687. }
  688. static void blkio_get_key_name(enum stat_sub_type type, const char *dname,
  689. char *str, int chars_left, bool diskname_only)
  690. {
  691. snprintf(str, chars_left, "%s", dname);
  692. chars_left -= strlen(str);
  693. if (chars_left <= 0) {
  694. printk(KERN_WARNING
  695. "Possibly incorrect cgroup stat display format");
  696. return;
  697. }
  698. if (diskname_only)
  699. return;
  700. switch (type) {
  701. case BLKIO_STAT_READ:
  702. strlcat(str, " Read", chars_left);
  703. break;
  704. case BLKIO_STAT_WRITE:
  705. strlcat(str, " Write", chars_left);
  706. break;
  707. case BLKIO_STAT_SYNC:
  708. strlcat(str, " Sync", chars_left);
  709. break;
  710. case BLKIO_STAT_ASYNC:
  711. strlcat(str, " Async", chars_left);
  712. break;
  713. case BLKIO_STAT_TOTAL:
  714. strlcat(str, " Total", chars_left);
  715. break;
  716. default:
  717. strlcat(str, " Invalid", chars_left);
  718. }
  719. }
  720. static uint64_t blkio_read_stat_cpu(struct blkio_group *blkg, int plid,
  721. enum stat_type_cpu type, enum stat_sub_type sub_type)
  722. {
  723. struct blkg_policy_data *pd = blkg->pd[plid];
  724. int cpu;
  725. struct blkio_group_stats_cpu *stats_cpu;
  726. u64 val = 0, tval;
  727. if (pd->stats_cpu == NULL)
  728. return val;
  729. for_each_possible_cpu(cpu) {
  730. unsigned int start;
  731. stats_cpu = per_cpu_ptr(pd->stats_cpu, cpu);
  732. do {
  733. start = u64_stats_fetch_begin(&stats_cpu->syncp);
  734. if (type == BLKIO_STAT_CPU_SECTORS)
  735. tval = stats_cpu->sectors;
  736. else
  737. tval = stats_cpu->stat_arr_cpu[type][sub_type];
  738. } while(u64_stats_fetch_retry(&stats_cpu->syncp, start));
  739. val += tval;
  740. }
  741. return val;
  742. }
  743. static uint64_t blkio_get_stat_cpu(struct blkio_group *blkg, int plid,
  744. struct cgroup_map_cb *cb, const char *dname,
  745. enum stat_type_cpu type)
  746. {
  747. uint64_t disk_total, val;
  748. char key_str[MAX_KEY_LEN];
  749. enum stat_sub_type sub_type;
  750. if (type == BLKIO_STAT_CPU_SECTORS) {
  751. val = blkio_read_stat_cpu(blkg, plid, type, 0);
  752. blkio_get_key_name(0, dname, key_str, MAX_KEY_LEN, true);
  753. cb->fill(cb, key_str, val);
  754. return val;
  755. }
  756. for (sub_type = BLKIO_STAT_READ; sub_type < BLKIO_STAT_TOTAL;
  757. sub_type++) {
  758. blkio_get_key_name(sub_type, dname, key_str, MAX_KEY_LEN,
  759. false);
  760. val = blkio_read_stat_cpu(blkg, plid, type, sub_type);
  761. cb->fill(cb, key_str, val);
  762. }
  763. disk_total = blkio_read_stat_cpu(blkg, plid, type, BLKIO_STAT_READ) +
  764. blkio_read_stat_cpu(blkg, plid, type, BLKIO_STAT_WRITE);
  765. blkio_get_key_name(BLKIO_STAT_TOTAL, dname, key_str, MAX_KEY_LEN,
  766. false);
  767. cb->fill(cb, key_str, disk_total);
  768. return disk_total;
  769. }
  770. static uint64_t blkio_get_stat(struct blkio_group *blkg, int plid,
  771. struct cgroup_map_cb *cb, const char *dname,
  772. enum stat_type type)
  773. {
  774. struct blkio_group_stats *stats = &blkg->pd[plid]->stats;
  775. uint64_t v = 0, disk_total = 0;
  776. char key_str[MAX_KEY_LEN];
  777. unsigned int sync_start;
  778. int st;
  779. if (type >= BLKIO_STAT_ARR_NR) {
  780. do {
  781. sync_start = u64_stats_fetch_begin(&stats->syncp);
  782. switch (type) {
  783. case BLKIO_STAT_TIME:
  784. v = stats->time;
  785. break;
  786. #ifdef CONFIG_DEBUG_BLK_CGROUP
  787. case BLKIO_STAT_UNACCOUNTED_TIME:
  788. v = stats->unaccounted_time;
  789. break;
  790. case BLKIO_STAT_AVG_QUEUE_SIZE: {
  791. uint64_t samples = stats->avg_queue_size_samples;
  792. if (samples) {
  793. v = stats->avg_queue_size_sum;
  794. do_div(v, samples);
  795. }
  796. break;
  797. }
  798. case BLKIO_STAT_IDLE_TIME:
  799. v = stats->idle_time;
  800. break;
  801. case BLKIO_STAT_EMPTY_TIME:
  802. v = stats->empty_time;
  803. break;
  804. case BLKIO_STAT_DEQUEUE:
  805. v = stats->dequeue;
  806. break;
  807. case BLKIO_STAT_GROUP_WAIT_TIME:
  808. v = stats->group_wait_time;
  809. break;
  810. #endif
  811. default:
  812. WARN_ON_ONCE(1);
  813. }
  814. } while (u64_stats_fetch_retry(&stats->syncp, sync_start));
  815. blkio_get_key_name(0, dname, key_str, MAX_KEY_LEN, true);
  816. cb->fill(cb, key_str, v);
  817. return v;
  818. }
  819. for (st = BLKIO_STAT_READ; st < BLKIO_STAT_TOTAL; st++) {
  820. do {
  821. sync_start = u64_stats_fetch_begin(&stats->syncp);
  822. v = stats->stat_arr[type][st];
  823. } while (u64_stats_fetch_retry(&stats->syncp, sync_start));
  824. blkio_get_key_name(st, dname, key_str, MAX_KEY_LEN, false);
  825. cb->fill(cb, key_str, v);
  826. if (st == BLKIO_STAT_READ || st == BLKIO_STAT_WRITE)
  827. disk_total += v;
  828. }
  829. blkio_get_key_name(BLKIO_STAT_TOTAL, dname, key_str, MAX_KEY_LEN,
  830. false);
  831. cb->fill(cb, key_str, disk_total);
  832. return disk_total;
  833. }
  834. static int blkio_policy_parse_and_set(char *buf, enum blkio_policy_id plid,
  835. int fileid, struct blkio_cgroup *blkcg)
  836. {
  837. struct gendisk *disk = NULL;
  838. struct blkio_group *blkg = NULL;
  839. struct blkg_policy_data *pd;
  840. char *s[4], *p, *major_s = NULL, *minor_s = NULL;
  841. unsigned long major, minor;
  842. int i = 0, ret = -EINVAL;
  843. int part;
  844. dev_t dev;
  845. u64 temp;
  846. memset(s, 0, sizeof(s));
  847. while ((p = strsep(&buf, " ")) != NULL) {
  848. if (!*p)
  849. continue;
  850. s[i++] = p;
  851. /* Prevent from inputing too many things */
  852. if (i == 3)
  853. break;
  854. }
  855. if (i != 2)
  856. goto out;
  857. p = strsep(&s[0], ":");
  858. if (p != NULL)
  859. major_s = p;
  860. else
  861. goto out;
  862. minor_s = s[0];
  863. if (!minor_s)
  864. goto out;
  865. if (strict_strtoul(major_s, 10, &major))
  866. goto out;
  867. if (strict_strtoul(minor_s, 10, &minor))
  868. goto out;
  869. dev = MKDEV(major, minor);
  870. if (strict_strtoull(s[1], 10, &temp))
  871. goto out;
  872. disk = get_gendisk(dev, &part);
  873. if (!disk || part)
  874. goto out;
  875. rcu_read_lock();
  876. spin_lock_irq(disk->queue->queue_lock);
  877. blkg = blkg_lookup_create(blkcg, disk->queue, plid, false);
  878. spin_unlock_irq(disk->queue->queue_lock);
  879. if (IS_ERR(blkg)) {
  880. ret = PTR_ERR(blkg);
  881. goto out_unlock;
  882. }
  883. pd = blkg->pd[plid];
  884. switch (plid) {
  885. case BLKIO_POLICY_PROP:
  886. if ((temp < BLKIO_WEIGHT_MIN && temp > 0) ||
  887. temp > BLKIO_WEIGHT_MAX)
  888. goto out_unlock;
  889. pd->conf.weight = temp;
  890. blkio_update_group_weight(blkg, plid, temp ?: blkcg->weight);
  891. break;
  892. case BLKIO_POLICY_THROTL:
  893. switch(fileid) {
  894. case BLKIO_THROTL_read_bps_device:
  895. pd->conf.bps[READ] = temp;
  896. blkio_update_group_bps(blkg, plid, temp ?: -1, fileid);
  897. break;
  898. case BLKIO_THROTL_write_bps_device:
  899. pd->conf.bps[WRITE] = temp;
  900. blkio_update_group_bps(blkg, plid, temp ?: -1, fileid);
  901. break;
  902. case BLKIO_THROTL_read_iops_device:
  903. if (temp > THROTL_IOPS_MAX)
  904. goto out_unlock;
  905. pd->conf.iops[READ] = temp;
  906. blkio_update_group_iops(blkg, plid, temp ?: -1, fileid);
  907. break;
  908. case BLKIO_THROTL_write_iops_device:
  909. if (temp > THROTL_IOPS_MAX)
  910. goto out_unlock;
  911. pd->conf.iops[WRITE] = temp;
  912. blkio_update_group_iops(blkg, plid, temp ?: -1, fileid);
  913. break;
  914. }
  915. break;
  916. default:
  917. BUG();
  918. }
  919. ret = 0;
  920. out_unlock:
  921. rcu_read_unlock();
  922. out:
  923. put_disk(disk);
  924. /*
  925. * If queue was bypassing, we should retry. Do so after a short
  926. * msleep(). It isn't strictly necessary but queue can be
  927. * bypassing for some time and it's always nice to avoid busy
  928. * looping.
  929. */
  930. if (ret == -EBUSY) {
  931. msleep(10);
  932. return restart_syscall();
  933. }
  934. return ret;
  935. }
  936. static int blkiocg_file_write(struct cgroup *cgrp, struct cftype *cft,
  937. const char *buffer)
  938. {
  939. int ret = 0;
  940. char *buf;
  941. struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);
  942. enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
  943. int fileid = BLKIOFILE_ATTR(cft->private);
  944. buf = kstrdup(buffer, GFP_KERNEL);
  945. if (!buf)
  946. return -ENOMEM;
  947. ret = blkio_policy_parse_and_set(buf, plid, fileid, blkcg);
  948. kfree(buf);
  949. return ret;
  950. }
  951. static const char *blkg_dev_name(struct blkio_group *blkg)
  952. {
  953. /* some drivers (floppy) instantiate a queue w/o disk registered */
  954. if (blkg->q->backing_dev_info.dev)
  955. return dev_name(blkg->q->backing_dev_info.dev);
  956. return NULL;
  957. }
  958. static void blkio_print_group_conf(struct cftype *cft, struct blkio_group *blkg,
  959. struct seq_file *m)
  960. {
  961. int plid = BLKIOFILE_POLICY(cft->private);
  962. int fileid = BLKIOFILE_ATTR(cft->private);
  963. struct blkg_policy_data *pd = blkg->pd[plid];
  964. const char *dname = blkg_dev_name(blkg);
  965. int rw = WRITE;
  966. if (!dname)
  967. return;
  968. switch (plid) {
  969. case BLKIO_POLICY_PROP:
  970. if (pd->conf.weight)
  971. seq_printf(m, "%s\t%u\n",
  972. dname, pd->conf.weight);
  973. break;
  974. case BLKIO_POLICY_THROTL:
  975. switch (fileid) {
  976. case BLKIO_THROTL_read_bps_device:
  977. rw = READ;
  978. case BLKIO_THROTL_write_bps_device:
  979. if (pd->conf.bps[rw])
  980. seq_printf(m, "%s\t%llu\n",
  981. dname, pd->conf.bps[rw]);
  982. break;
  983. case BLKIO_THROTL_read_iops_device:
  984. rw = READ;
  985. case BLKIO_THROTL_write_iops_device:
  986. if (pd->conf.iops[rw])
  987. seq_printf(m, "%s\t%u\n",
  988. dname, pd->conf.iops[rw]);
  989. break;
  990. }
  991. break;
  992. default:
  993. BUG();
  994. }
  995. }
  996. /* cgroup files which read their data from policy nodes end up here */
  997. static void blkio_read_conf(struct cftype *cft, struct blkio_cgroup *blkcg,
  998. struct seq_file *m)
  999. {
  1000. struct blkio_group *blkg;
  1001. struct hlist_node *n;
  1002. spin_lock_irq(&blkcg->lock);
  1003. hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node)
  1004. blkio_print_group_conf(cft, blkg, m);
  1005. spin_unlock_irq(&blkcg->lock);
  1006. }
  1007. static int blkiocg_file_read(struct cgroup *cgrp, struct cftype *cft,
  1008. struct seq_file *m)
  1009. {
  1010. struct blkio_cgroup *blkcg;
  1011. enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
  1012. int name = BLKIOFILE_ATTR(cft->private);
  1013. blkcg = cgroup_to_blkio_cgroup(cgrp);
  1014. switch(plid) {
  1015. case BLKIO_POLICY_PROP:
  1016. switch(name) {
  1017. case BLKIO_PROP_weight_device:
  1018. blkio_read_conf(cft, blkcg, m);
  1019. return 0;
  1020. default:
  1021. BUG();
  1022. }
  1023. break;
  1024. case BLKIO_POLICY_THROTL:
  1025. switch(name){
  1026. case BLKIO_THROTL_read_bps_device:
  1027. case BLKIO_THROTL_write_bps_device:
  1028. case BLKIO_THROTL_read_iops_device:
  1029. case BLKIO_THROTL_write_iops_device:
  1030. blkio_read_conf(cft, blkcg, m);
  1031. return 0;
  1032. default:
  1033. BUG();
  1034. }
  1035. break;
  1036. default:
  1037. BUG();
  1038. }
  1039. return 0;
  1040. }
  1041. static int blkio_read_blkg_stats(struct blkio_cgroup *blkcg,
  1042. struct cftype *cft, struct cgroup_map_cb *cb,
  1043. enum stat_type type, bool show_total, bool pcpu)
  1044. {
  1045. struct blkio_group *blkg;
  1046. struct hlist_node *n;
  1047. uint64_t cgroup_total = 0;
  1048. spin_lock_irq(&blkcg->lock);
  1049. hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
  1050. const char *dname = blkg_dev_name(blkg);
  1051. int plid = BLKIOFILE_POLICY(cft->private);
  1052. if (!dname)
  1053. continue;
  1054. if (pcpu)
  1055. cgroup_total += blkio_get_stat_cpu(blkg, plid,
  1056. cb, dname, type);
  1057. else
  1058. cgroup_total += blkio_get_stat(blkg, plid,
  1059. cb, dname, type);
  1060. }
  1061. if (show_total)
  1062. cb->fill(cb, "Total", cgroup_total);
  1063. spin_unlock_irq(&blkcg->lock);
  1064. return 0;
  1065. }
  1066. /* All map kind of cgroup file get serviced by this function */
  1067. static int blkiocg_file_read_map(struct cgroup *cgrp, struct cftype *cft,
  1068. struct cgroup_map_cb *cb)
  1069. {
  1070. struct blkio_cgroup *blkcg;
  1071. enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
  1072. int name = BLKIOFILE_ATTR(cft->private);
  1073. blkcg = cgroup_to_blkio_cgroup(cgrp);
  1074. switch(plid) {
  1075. case BLKIO_POLICY_PROP:
  1076. switch(name) {
  1077. case BLKIO_PROP_time:
  1078. return blkio_read_blkg_stats(blkcg, cft, cb,
  1079. BLKIO_STAT_TIME, 0, 0);
  1080. case BLKIO_PROP_sectors:
  1081. return blkio_read_blkg_stats(blkcg, cft, cb,
  1082. BLKIO_STAT_CPU_SECTORS, 0, 1);
  1083. case BLKIO_PROP_io_service_bytes:
  1084. return blkio_read_blkg_stats(blkcg, cft, cb,
  1085. BLKIO_STAT_CPU_SERVICE_BYTES, 1, 1);
  1086. case BLKIO_PROP_io_serviced:
  1087. return blkio_read_blkg_stats(blkcg, cft, cb,
  1088. BLKIO_STAT_CPU_SERVICED, 1, 1);
  1089. case BLKIO_PROP_io_service_time:
  1090. return blkio_read_blkg_stats(blkcg, cft, cb,
  1091. BLKIO_STAT_SERVICE_TIME, 1, 0);
  1092. case BLKIO_PROP_io_wait_time:
  1093. return blkio_read_blkg_stats(blkcg, cft, cb,
  1094. BLKIO_STAT_WAIT_TIME, 1, 0);
  1095. case BLKIO_PROP_io_merged:
  1096. return blkio_read_blkg_stats(blkcg, cft, cb,
  1097. BLKIO_STAT_MERGED, 1, 0);
  1098. case BLKIO_PROP_io_queued:
  1099. return blkio_read_blkg_stats(blkcg, cft, cb,
  1100. BLKIO_STAT_QUEUED, 1, 0);
  1101. #ifdef CONFIG_DEBUG_BLK_CGROUP
  1102. case BLKIO_PROP_unaccounted_time:
  1103. return blkio_read_blkg_stats(blkcg, cft, cb,
  1104. BLKIO_STAT_UNACCOUNTED_TIME, 0, 0);
  1105. case BLKIO_PROP_dequeue:
  1106. return blkio_read_blkg_stats(blkcg, cft, cb,
  1107. BLKIO_STAT_DEQUEUE, 0, 0);
  1108. case BLKIO_PROP_avg_queue_size:
  1109. return blkio_read_blkg_stats(blkcg, cft, cb,
  1110. BLKIO_STAT_AVG_QUEUE_SIZE, 0, 0);
  1111. case BLKIO_PROP_group_wait_time:
  1112. return blkio_read_blkg_stats(blkcg, cft, cb,
  1113. BLKIO_STAT_GROUP_WAIT_TIME, 0, 0);
  1114. case BLKIO_PROP_idle_time:
  1115. return blkio_read_blkg_stats(blkcg, cft, cb,
  1116. BLKIO_STAT_IDLE_TIME, 0, 0);
  1117. case BLKIO_PROP_empty_time:
  1118. return blkio_read_blkg_stats(blkcg, cft, cb,
  1119. BLKIO_STAT_EMPTY_TIME, 0, 0);
  1120. #endif
  1121. default:
  1122. BUG();
  1123. }
  1124. break;
  1125. case BLKIO_POLICY_THROTL:
  1126. switch(name){
  1127. case BLKIO_THROTL_io_service_bytes:
  1128. return blkio_read_blkg_stats(blkcg, cft, cb,
  1129. BLKIO_STAT_CPU_SERVICE_BYTES, 1, 1);
  1130. case BLKIO_THROTL_io_serviced:
  1131. return blkio_read_blkg_stats(blkcg, cft, cb,
  1132. BLKIO_STAT_CPU_SERVICED, 1, 1);
  1133. default:
  1134. BUG();
  1135. }
  1136. break;
  1137. default:
  1138. BUG();
  1139. }
  1140. return 0;
  1141. }
  1142. static int blkio_weight_write(struct blkio_cgroup *blkcg, int plid, u64 val)
  1143. {
  1144. struct blkio_group *blkg;
  1145. struct hlist_node *n;
  1146. if (val < BLKIO_WEIGHT_MIN || val > BLKIO_WEIGHT_MAX)
  1147. return -EINVAL;
  1148. spin_lock(&blkio_list_lock);
  1149. spin_lock_irq(&blkcg->lock);
  1150. blkcg->weight = (unsigned int)val;
  1151. hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
  1152. struct blkg_policy_data *pd = blkg->pd[plid];
  1153. if (!pd->conf.weight)
  1154. blkio_update_group_weight(blkg, plid, blkcg->weight);
  1155. }
  1156. spin_unlock_irq(&blkcg->lock);
  1157. spin_unlock(&blkio_list_lock);
  1158. return 0;
  1159. }
  1160. static u64 blkiocg_file_read_u64 (struct cgroup *cgrp, struct cftype *cft) {
  1161. struct blkio_cgroup *blkcg;
  1162. enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
  1163. int name = BLKIOFILE_ATTR(cft->private);
  1164. blkcg = cgroup_to_blkio_cgroup(cgrp);
  1165. switch(plid) {
  1166. case BLKIO_POLICY_PROP:
  1167. switch(name) {
  1168. case BLKIO_PROP_weight:
  1169. return (u64)blkcg->weight;
  1170. }
  1171. break;
  1172. default:
  1173. BUG();
  1174. }
  1175. return 0;
  1176. }
  1177. static int
  1178. blkiocg_file_write_u64(struct cgroup *cgrp, struct cftype *cft, u64 val)
  1179. {
  1180. struct blkio_cgroup *blkcg;
  1181. enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
  1182. int name = BLKIOFILE_ATTR(cft->private);
  1183. blkcg = cgroup_to_blkio_cgroup(cgrp);
  1184. switch(plid) {
  1185. case BLKIO_POLICY_PROP:
  1186. switch(name) {
  1187. case BLKIO_PROP_weight:
  1188. return blkio_weight_write(blkcg, plid, val);
  1189. }
  1190. break;
  1191. default:
  1192. BUG();
  1193. }
  1194. return 0;
  1195. }
  1196. struct cftype blkio_files[] = {
  1197. {
  1198. .name = "weight_device",
  1199. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1200. BLKIO_PROP_weight_device),
  1201. .read_seq_string = blkiocg_file_read,
  1202. .write_string = blkiocg_file_write,
  1203. .max_write_len = 256,
  1204. },
  1205. {
  1206. .name = "weight",
  1207. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1208. BLKIO_PROP_weight),
  1209. .read_u64 = blkiocg_file_read_u64,
  1210. .write_u64 = blkiocg_file_write_u64,
  1211. },
  1212. {
  1213. .name = "time",
  1214. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1215. BLKIO_PROP_time),
  1216. .read_map = blkiocg_file_read_map,
  1217. },
  1218. {
  1219. .name = "sectors",
  1220. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1221. BLKIO_PROP_sectors),
  1222. .read_map = blkiocg_file_read_map,
  1223. },
  1224. {
  1225. .name = "io_service_bytes",
  1226. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1227. BLKIO_PROP_io_service_bytes),
  1228. .read_map = blkiocg_file_read_map,
  1229. },
  1230. {
  1231. .name = "io_serviced",
  1232. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1233. BLKIO_PROP_io_serviced),
  1234. .read_map = blkiocg_file_read_map,
  1235. },
  1236. {
  1237. .name = "io_service_time",
  1238. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1239. BLKIO_PROP_io_service_time),
  1240. .read_map = blkiocg_file_read_map,
  1241. },
  1242. {
  1243. .name = "io_wait_time",
  1244. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1245. BLKIO_PROP_io_wait_time),
  1246. .read_map = blkiocg_file_read_map,
  1247. },
  1248. {
  1249. .name = "io_merged",
  1250. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1251. BLKIO_PROP_io_merged),
  1252. .read_map = blkiocg_file_read_map,
  1253. },
  1254. {
  1255. .name = "io_queued",
  1256. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1257. BLKIO_PROP_io_queued),
  1258. .read_map = blkiocg_file_read_map,
  1259. },
  1260. {
  1261. .name = "reset_stats",
  1262. .write_u64 = blkiocg_reset_stats,
  1263. },
  1264. #ifdef CONFIG_BLK_DEV_THROTTLING
  1265. {
  1266. .name = "throttle.read_bps_device",
  1267. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
  1268. BLKIO_THROTL_read_bps_device),
  1269. .read_seq_string = blkiocg_file_read,
  1270. .write_string = blkiocg_file_write,
  1271. .max_write_len = 256,
  1272. },
  1273. {
  1274. .name = "throttle.write_bps_device",
  1275. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
  1276. BLKIO_THROTL_write_bps_device),
  1277. .read_seq_string = blkiocg_file_read,
  1278. .write_string = blkiocg_file_write,
  1279. .max_write_len = 256,
  1280. },
  1281. {
  1282. .name = "throttle.read_iops_device",
  1283. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
  1284. BLKIO_THROTL_read_iops_device),
  1285. .read_seq_string = blkiocg_file_read,
  1286. .write_string = blkiocg_file_write,
  1287. .max_write_len = 256,
  1288. },
  1289. {
  1290. .name = "throttle.write_iops_device",
  1291. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
  1292. BLKIO_THROTL_write_iops_device),
  1293. .read_seq_string = blkiocg_file_read,
  1294. .write_string = blkiocg_file_write,
  1295. .max_write_len = 256,
  1296. },
  1297. {
  1298. .name = "throttle.io_service_bytes",
  1299. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
  1300. BLKIO_THROTL_io_service_bytes),
  1301. .read_map = blkiocg_file_read_map,
  1302. },
  1303. {
  1304. .name = "throttle.io_serviced",
  1305. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
  1306. BLKIO_THROTL_io_serviced),
  1307. .read_map = blkiocg_file_read_map,
  1308. },
  1309. #endif /* CONFIG_BLK_DEV_THROTTLING */
  1310. #ifdef CONFIG_DEBUG_BLK_CGROUP
  1311. {
  1312. .name = "avg_queue_size",
  1313. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1314. BLKIO_PROP_avg_queue_size),
  1315. .read_map = blkiocg_file_read_map,
  1316. },
  1317. {
  1318. .name = "group_wait_time",
  1319. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1320. BLKIO_PROP_group_wait_time),
  1321. .read_map = blkiocg_file_read_map,
  1322. },
  1323. {
  1324. .name = "idle_time",
  1325. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1326. BLKIO_PROP_idle_time),
  1327. .read_map = blkiocg_file_read_map,
  1328. },
  1329. {
  1330. .name = "empty_time",
  1331. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1332. BLKIO_PROP_empty_time),
  1333. .read_map = blkiocg_file_read_map,
  1334. },
  1335. {
  1336. .name = "dequeue",
  1337. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1338. BLKIO_PROP_dequeue),
  1339. .read_map = blkiocg_file_read_map,
  1340. },
  1341. {
  1342. .name = "unaccounted_time",
  1343. .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
  1344. BLKIO_PROP_unaccounted_time),
  1345. .read_map = blkiocg_file_read_map,
  1346. },
  1347. #endif
  1348. };
  1349. static int blkiocg_populate(struct cgroup_subsys *subsys, struct cgroup *cgroup)
  1350. {
  1351. return cgroup_add_files(cgroup, subsys, blkio_files,
  1352. ARRAY_SIZE(blkio_files));
  1353. }
  1354. /**
  1355. * blkiocg_pre_destroy - cgroup pre_destroy callback
  1356. * @subsys: cgroup subsys
  1357. * @cgroup: cgroup of interest
  1358. *
  1359. * This function is called when @cgroup is about to go away and responsible
  1360. * for shooting down all blkgs associated with @cgroup. blkgs should be
  1361. * removed while holding both q and blkcg locks. As blkcg lock is nested
  1362. * inside q lock, this function performs reverse double lock dancing.
  1363. *
  1364. * This is the blkcg counterpart of ioc_release_fn().
  1365. */
  1366. static int blkiocg_pre_destroy(struct cgroup_subsys *subsys,
  1367. struct cgroup *cgroup)
  1368. {
  1369. struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgroup);
  1370. spin_lock_irq(&blkcg->lock);
  1371. while (!hlist_empty(&blkcg->blkg_list)) {
  1372. struct blkio_group *blkg = hlist_entry(blkcg->blkg_list.first,
  1373. struct blkio_group, blkcg_node);
  1374. struct request_queue *q = blkg->q;
  1375. if (spin_trylock(q->queue_lock)) {
  1376. blkg_destroy(blkg);
  1377. spin_unlock(q->queue_lock);
  1378. } else {
  1379. spin_unlock_irq(&blkcg->lock);
  1380. cpu_relax();
  1381. spin_lock(&blkcg->lock);
  1382. }
  1383. }
  1384. spin_unlock_irq(&blkcg->lock);
  1385. return 0;
  1386. }
  1387. static void blkiocg_destroy(struct cgroup_subsys *subsys, struct cgroup *cgroup)
  1388. {
  1389. struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgroup);
  1390. if (blkcg != &blkio_root_cgroup)
  1391. kfree(blkcg);
  1392. }
  1393. static struct cgroup_subsys_state *
  1394. blkiocg_create(struct cgroup_subsys *subsys, struct cgroup *cgroup)
  1395. {
  1396. static atomic64_t id_seq = ATOMIC64_INIT(0);
  1397. struct blkio_cgroup *blkcg;
  1398. struct cgroup *parent = cgroup->parent;
  1399. if (!parent) {
  1400. blkcg = &blkio_root_cgroup;
  1401. goto done;
  1402. }
  1403. blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
  1404. if (!blkcg)
  1405. return ERR_PTR(-ENOMEM);
  1406. blkcg->weight = BLKIO_WEIGHT_DEFAULT;
  1407. blkcg->id = atomic64_inc_return(&id_seq); /* root is 0, start from 1 */
  1408. done:
  1409. spin_lock_init(&blkcg->lock);
  1410. INIT_HLIST_HEAD(&blkcg->blkg_list);
  1411. return &blkcg->css;
  1412. }
  1413. /**
  1414. * blkcg_init_queue - initialize blkcg part of request queue
  1415. * @q: request_queue to initialize
  1416. *
  1417. * Called from blk_alloc_queue_node(). Responsible for initializing blkcg
  1418. * part of new request_queue @q.
  1419. *
  1420. * RETURNS:
  1421. * 0 on success, -errno on failure.
  1422. */
  1423. int blkcg_init_queue(struct request_queue *q)
  1424. {
  1425. int ret;
  1426. might_sleep();
  1427. ret = blk_throtl_init(q);
  1428. if (ret)
  1429. return ret;
  1430. mutex_lock(&all_q_mutex);
  1431. INIT_LIST_HEAD(&q->all_q_node);
  1432. list_add_tail(&q->all_q_node, &all_q_list);
  1433. mutex_unlock(&all_q_mutex);
  1434. return 0;
  1435. }
  1436. /**
  1437. * blkcg_drain_queue - drain blkcg part of request_queue
  1438. * @q: request_queue to drain
  1439. *
  1440. * Called from blk_drain_queue(). Responsible for draining blkcg part.
  1441. */
  1442. void blkcg_drain_queue(struct request_queue *q)
  1443. {
  1444. lockdep_assert_held(q->queue_lock);
  1445. blk_throtl_drain(q);
  1446. }
  1447. /**
  1448. * blkcg_exit_queue - exit and release blkcg part of request_queue
  1449. * @q: request_queue being released
  1450. *
  1451. * Called from blk_release_queue(). Responsible for exiting blkcg part.
  1452. */
  1453. void blkcg_exit_queue(struct request_queue *q)
  1454. {
  1455. mutex_lock(&all_q_mutex);
  1456. list_del_init(&q->all_q_node);
  1457. mutex_unlock(&all_q_mutex);
  1458. blkg_destroy_all(q, true);
  1459. blk_throtl_exit(q);
  1460. }
  1461. /*
  1462. * We cannot support shared io contexts, as we have no mean to support
  1463. * two tasks with the same ioc in two different groups without major rework
  1464. * of the main cic data structures. For now we allow a task to change
  1465. * its cgroup only if it's the only owner of its ioc.
  1466. */
  1467. static int blkiocg_can_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
  1468. struct cgroup_taskset *tset)
  1469. {
  1470. struct task_struct *task;
  1471. struct io_context *ioc;
  1472. int ret = 0;
  1473. /* task_lock() is needed to avoid races with exit_io_context() */
  1474. cgroup_taskset_for_each(task, cgrp, tset) {
  1475. task_lock(task);
  1476. ioc = task->io_context;
  1477. if (ioc && atomic_read(&ioc->nr_tasks) > 1)
  1478. ret = -EINVAL;
  1479. task_unlock(task);
  1480. if (ret)
  1481. break;
  1482. }
  1483. return ret;
  1484. }
  1485. static void blkiocg_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
  1486. struct cgroup_taskset *tset)
  1487. {
  1488. struct task_struct *task;
  1489. struct io_context *ioc;
  1490. cgroup_taskset_for_each(task, cgrp, tset) {
  1491. /* we don't lose anything even if ioc allocation fails */
  1492. ioc = get_task_io_context(task, GFP_ATOMIC, NUMA_NO_NODE);
  1493. if (ioc) {
  1494. ioc_cgroup_changed(ioc);
  1495. put_io_context(ioc);
  1496. }
  1497. }
  1498. }
  1499. static void blkcg_bypass_start(void)
  1500. __acquires(&all_q_mutex)
  1501. {
  1502. struct request_queue *q;
  1503. mutex_lock(&all_q_mutex);
  1504. list_for_each_entry(q, &all_q_list, all_q_node) {
  1505. blk_queue_bypass_start(q);
  1506. blkg_destroy_all(q, false);
  1507. }
  1508. }
  1509. static void blkcg_bypass_end(void)
  1510. __releases(&all_q_mutex)
  1511. {
  1512. struct request_queue *q;
  1513. list_for_each_entry(q, &all_q_list, all_q_node)
  1514. blk_queue_bypass_end(q);
  1515. mutex_unlock(&all_q_mutex);
  1516. }
  1517. void blkio_policy_register(struct blkio_policy_type *blkiop)
  1518. {
  1519. struct request_queue *q;
  1520. blkcg_bypass_start();
  1521. spin_lock(&blkio_list_lock);
  1522. BUG_ON(blkio_policy[blkiop->plid]);
  1523. blkio_policy[blkiop->plid] = blkiop;
  1524. list_add_tail(&blkiop->list, &blkio_list);
  1525. spin_unlock(&blkio_list_lock);
  1526. list_for_each_entry(q, &all_q_list, all_q_node)
  1527. update_root_blkg_pd(q, blkiop->plid);
  1528. blkcg_bypass_end();
  1529. }
  1530. EXPORT_SYMBOL_GPL(blkio_policy_register);
  1531. void blkio_policy_unregister(struct blkio_policy_type *blkiop)
  1532. {
  1533. struct request_queue *q;
  1534. blkcg_bypass_start();
  1535. spin_lock(&blkio_list_lock);
  1536. BUG_ON(blkio_policy[blkiop->plid] != blkiop);
  1537. blkio_policy[blkiop->plid] = NULL;
  1538. list_del_init(&blkiop->list);
  1539. spin_unlock(&blkio_list_lock);
  1540. list_for_each_entry(q, &all_q_list, all_q_node)
  1541. update_root_blkg_pd(q, blkiop->plid);
  1542. blkcg_bypass_end();
  1543. }
  1544. EXPORT_SYMBOL_GPL(blkio_policy_unregister);