request.c 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338
  1. /*
  2. * Main bcache entry point - handle a read or a write request and decide what to
  3. * do with it; the make_request functions are called by the block layer.
  4. *
  5. * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
  6. * Copyright 2012 Google, Inc.
  7. */
  8. #include "bcache.h"
  9. #include "btree.h"
  10. #include "debug.h"
  11. #include "request.h"
  12. #include "writeback.h"
  13. #include <linux/cgroup.h>
  14. #include <linux/module.h>
  15. #include <linux/hash.h>
  16. #include <linux/random.h>
  17. #include "blk-cgroup.h"
  18. #include <trace/events/bcache.h>
  19. #define CUTOFF_CACHE_ADD 95
  20. #define CUTOFF_CACHE_READA 90
  21. struct kmem_cache *bch_search_cache;
  22. static void bch_data_insert_start(struct closure *);
  23. /* Cgroup interface */
  24. #ifdef CONFIG_CGROUP_BCACHE
  25. static struct bch_cgroup bcache_default_cgroup = { .cache_mode = -1 };
  26. static struct bch_cgroup *cgroup_to_bcache(struct cgroup *cgroup)
  27. {
  28. struct cgroup_subsys_state *css;
  29. return cgroup &&
  30. (css = cgroup_subsys_state(cgroup, bcache_subsys_id))
  31. ? container_of(css, struct bch_cgroup, css)
  32. : &bcache_default_cgroup;
  33. }
  34. struct bch_cgroup *bch_bio_to_cgroup(struct bio *bio)
  35. {
  36. struct cgroup_subsys_state *css = bio->bi_css
  37. ? cgroup_subsys_state(bio->bi_css->cgroup, bcache_subsys_id)
  38. : task_subsys_state(current, bcache_subsys_id);
  39. return css
  40. ? container_of(css, struct bch_cgroup, css)
  41. : &bcache_default_cgroup;
  42. }
  43. static ssize_t cache_mode_read(struct cgroup *cgrp, struct cftype *cft,
  44. struct file *file,
  45. char __user *buf, size_t nbytes, loff_t *ppos)
  46. {
  47. char tmp[1024];
  48. int len = bch_snprint_string_list(tmp, PAGE_SIZE, bch_cache_modes,
  49. cgroup_to_bcache(cgrp)->cache_mode + 1);
  50. if (len < 0)
  51. return len;
  52. return simple_read_from_buffer(buf, nbytes, ppos, tmp, len);
  53. }
  54. static int cache_mode_write(struct cgroup *cgrp, struct cftype *cft,
  55. const char *buf)
  56. {
  57. int v = bch_read_string_list(buf, bch_cache_modes);
  58. if (v < 0)
  59. return v;
  60. cgroup_to_bcache(cgrp)->cache_mode = v - 1;
  61. return 0;
  62. }
  63. static u64 bch_verify_read(struct cgroup *cgrp, struct cftype *cft)
  64. {
  65. return cgroup_to_bcache(cgrp)->verify;
  66. }
  67. static int bch_verify_write(struct cgroup *cgrp, struct cftype *cft, u64 val)
  68. {
  69. cgroup_to_bcache(cgrp)->verify = val;
  70. return 0;
  71. }
  72. static u64 bch_cache_hits_read(struct cgroup *cgrp, struct cftype *cft)
  73. {
  74. struct bch_cgroup *bcachecg = cgroup_to_bcache(cgrp);
  75. return atomic_read(&bcachecg->stats.cache_hits);
  76. }
  77. static u64 bch_cache_misses_read(struct cgroup *cgrp, struct cftype *cft)
  78. {
  79. struct bch_cgroup *bcachecg = cgroup_to_bcache(cgrp);
  80. return atomic_read(&bcachecg->stats.cache_misses);
  81. }
  82. static u64 bch_cache_bypass_hits_read(struct cgroup *cgrp,
  83. struct cftype *cft)
  84. {
  85. struct bch_cgroup *bcachecg = cgroup_to_bcache(cgrp);
  86. return atomic_read(&bcachecg->stats.cache_bypass_hits);
  87. }
  88. static u64 bch_cache_bypass_misses_read(struct cgroup *cgrp,
  89. struct cftype *cft)
  90. {
  91. struct bch_cgroup *bcachecg = cgroup_to_bcache(cgrp);
  92. return atomic_read(&bcachecg->stats.cache_bypass_misses);
  93. }
  94. static struct cftype bch_files[] = {
  95. {
  96. .name = "cache_mode",
  97. .read = cache_mode_read,
  98. .write_string = cache_mode_write,
  99. },
  100. {
  101. .name = "verify",
  102. .read_u64 = bch_verify_read,
  103. .write_u64 = bch_verify_write,
  104. },
  105. {
  106. .name = "cache_hits",
  107. .read_u64 = bch_cache_hits_read,
  108. },
  109. {
  110. .name = "cache_misses",
  111. .read_u64 = bch_cache_misses_read,
  112. },
  113. {
  114. .name = "cache_bypass_hits",
  115. .read_u64 = bch_cache_bypass_hits_read,
  116. },
  117. {
  118. .name = "cache_bypass_misses",
  119. .read_u64 = bch_cache_bypass_misses_read,
  120. },
  121. { } /* terminate */
  122. };
  123. static void init_bch_cgroup(struct bch_cgroup *cg)
  124. {
  125. cg->cache_mode = -1;
  126. }
  127. static struct cgroup_subsys_state *bcachecg_create(struct cgroup *cgroup)
  128. {
  129. struct bch_cgroup *cg;
  130. cg = kzalloc(sizeof(*cg), GFP_KERNEL);
  131. if (!cg)
  132. return ERR_PTR(-ENOMEM);
  133. init_bch_cgroup(cg);
  134. return &cg->css;
  135. }
  136. static void bcachecg_destroy(struct cgroup *cgroup)
  137. {
  138. struct bch_cgroup *cg = cgroup_to_bcache(cgroup);
  139. free_css_id(&bcache_subsys, &cg->css);
  140. kfree(cg);
  141. }
  142. struct cgroup_subsys bcache_subsys = {
  143. .create = bcachecg_create,
  144. .destroy = bcachecg_destroy,
  145. .subsys_id = bcache_subsys_id,
  146. .name = "bcache",
  147. .module = THIS_MODULE,
  148. };
  149. EXPORT_SYMBOL_GPL(bcache_subsys);
  150. #endif
  151. static unsigned cache_mode(struct cached_dev *dc, struct bio *bio)
  152. {
  153. #ifdef CONFIG_CGROUP_BCACHE
  154. int r = bch_bio_to_cgroup(bio)->cache_mode;
  155. if (r >= 0)
  156. return r;
  157. #endif
  158. return BDEV_CACHE_MODE(&dc->sb);
  159. }
  160. static bool verify(struct cached_dev *dc, struct bio *bio)
  161. {
  162. #ifdef CONFIG_CGROUP_BCACHE
  163. if (bch_bio_to_cgroup(bio)->verify)
  164. return true;
  165. #endif
  166. return dc->verify;
  167. }
  168. static void bio_csum(struct bio *bio, struct bkey *k)
  169. {
  170. struct bio_vec *bv;
  171. uint64_t csum = 0;
  172. int i;
  173. bio_for_each_segment(bv, bio, i) {
  174. void *d = kmap(bv->bv_page) + bv->bv_offset;
  175. csum = bch_crc64_update(csum, d, bv->bv_len);
  176. kunmap(bv->bv_page);
  177. }
  178. k->ptr[KEY_PTRS(k)] = csum & (~0ULL >> 1);
  179. }
  180. /* Insert data into cache */
  181. static void bch_data_insert_keys(struct closure *cl)
  182. {
  183. struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
  184. atomic_t *journal_ref = NULL;
  185. struct bkey *replace_key = op->replace ? &op->replace_key : NULL;
  186. int ret;
  187. /*
  188. * If we're looping, might already be waiting on
  189. * another journal write - can't wait on more than one journal write at
  190. * a time
  191. *
  192. * XXX: this looks wrong
  193. */
  194. #if 0
  195. while (atomic_read(&s->cl.remaining) & CLOSURE_WAITING)
  196. closure_sync(&s->cl);
  197. #endif
  198. if (!op->replace)
  199. journal_ref = bch_journal(op->c, &op->insert_keys,
  200. op->flush_journal ? cl : NULL);
  201. ret = bch_btree_insert(op->c, &op->insert_keys,
  202. journal_ref, replace_key);
  203. if (ret == -ESRCH) {
  204. op->replace_collision = true;
  205. } else if (ret) {
  206. op->error = -ENOMEM;
  207. op->insert_data_done = true;
  208. }
  209. if (journal_ref)
  210. atomic_dec_bug(journal_ref);
  211. if (!op->insert_data_done)
  212. continue_at(cl, bch_data_insert_start, bcache_wq);
  213. bch_keylist_free(&op->insert_keys);
  214. closure_return(cl);
  215. }
  216. static void bch_data_invalidate(struct closure *cl)
  217. {
  218. struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
  219. struct bio *bio = op->bio;
  220. pr_debug("invalidating %i sectors from %llu",
  221. bio_sectors(bio), (uint64_t) bio->bi_sector);
  222. while (bio_sectors(bio)) {
  223. unsigned sectors = min(bio_sectors(bio),
  224. 1U << (KEY_SIZE_BITS - 1));
  225. if (bch_keylist_realloc(&op->insert_keys, 0, op->c))
  226. goto out;
  227. bio->bi_sector += sectors;
  228. bio->bi_size -= sectors << 9;
  229. bch_keylist_add(&op->insert_keys,
  230. &KEY(op->inode, bio->bi_sector, sectors));
  231. }
  232. op->insert_data_done = true;
  233. bio_put(bio);
  234. out:
  235. continue_at(cl, bch_data_insert_keys, bcache_wq);
  236. }
  237. static void bch_data_insert_error(struct closure *cl)
  238. {
  239. struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
  240. /*
  241. * Our data write just errored, which means we've got a bunch of keys to
  242. * insert that point to data that wasn't succesfully written.
  243. *
  244. * We don't have to insert those keys but we still have to invalidate
  245. * that region of the cache - so, if we just strip off all the pointers
  246. * from the keys we'll accomplish just that.
  247. */
  248. struct bkey *src = op->insert_keys.keys, *dst = op->insert_keys.keys;
  249. while (src != op->insert_keys.top) {
  250. struct bkey *n = bkey_next(src);
  251. SET_KEY_PTRS(src, 0);
  252. memmove(dst, src, bkey_bytes(src));
  253. dst = bkey_next(dst);
  254. src = n;
  255. }
  256. op->insert_keys.top = dst;
  257. bch_data_insert_keys(cl);
  258. }
  259. static void bch_data_insert_endio(struct bio *bio, int error)
  260. {
  261. struct closure *cl = bio->bi_private;
  262. struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
  263. if (error) {
  264. /* TODO: We could try to recover from this. */
  265. if (op->writeback)
  266. op->error = error;
  267. else if (!op->replace)
  268. set_closure_fn(cl, bch_data_insert_error, bcache_wq);
  269. else
  270. set_closure_fn(cl, NULL, NULL);
  271. }
  272. bch_bbio_endio(op->c, bio, error, "writing data to cache");
  273. }
  274. static void bch_data_insert_start(struct closure *cl)
  275. {
  276. struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
  277. struct bio *bio = op->bio, *n;
  278. if (op->bypass)
  279. return bch_data_invalidate(cl);
  280. if (atomic_sub_return(bio_sectors(bio), &op->c->sectors_to_gc) < 0) {
  281. set_gc_sectors(op->c);
  282. wake_up_gc(op->c);
  283. }
  284. /*
  285. * Journal writes are marked REQ_FLUSH; if the original write was a
  286. * flush, it'll wait on the journal write.
  287. */
  288. bio->bi_rw &= ~(REQ_FLUSH|REQ_FUA);
  289. do {
  290. unsigned i;
  291. struct bkey *k;
  292. struct bio_set *split = op->c->bio_split;
  293. /* 1 for the device pointer and 1 for the chksum */
  294. if (bch_keylist_realloc(&op->insert_keys,
  295. 1 + (op->csum ? 1 : 0),
  296. op->c))
  297. continue_at(cl, bch_data_insert_keys, bcache_wq);
  298. k = op->insert_keys.top;
  299. bkey_init(k);
  300. SET_KEY_INODE(k, op->inode);
  301. SET_KEY_OFFSET(k, bio->bi_sector);
  302. if (!bch_alloc_sectors(op->c, k, bio_sectors(bio),
  303. op->write_point, op->write_prio,
  304. op->writeback))
  305. goto err;
  306. n = bch_bio_split(bio, KEY_SIZE(k), GFP_NOIO, split);
  307. n->bi_end_io = bch_data_insert_endio;
  308. n->bi_private = cl;
  309. if (op->writeback) {
  310. SET_KEY_DIRTY(k, true);
  311. for (i = 0; i < KEY_PTRS(k); i++)
  312. SET_GC_MARK(PTR_BUCKET(op->c, k, i),
  313. GC_MARK_DIRTY);
  314. }
  315. SET_KEY_CSUM(k, op->csum);
  316. if (KEY_CSUM(k))
  317. bio_csum(n, k);
  318. trace_bcache_cache_insert(k);
  319. bch_keylist_push(&op->insert_keys);
  320. n->bi_rw |= REQ_WRITE;
  321. bch_submit_bbio(n, op->c, k, 0);
  322. } while (n != bio);
  323. op->insert_data_done = true;
  324. continue_at(cl, bch_data_insert_keys, bcache_wq);
  325. err:
  326. /* bch_alloc_sectors() blocks if s->writeback = true */
  327. BUG_ON(op->writeback);
  328. /*
  329. * But if it's not a writeback write we'd rather just bail out if
  330. * there aren't any buckets ready to write to - it might take awhile and
  331. * we might be starving btree writes for gc or something.
  332. */
  333. if (!op->replace) {
  334. /*
  335. * Writethrough write: We can't complete the write until we've
  336. * updated the index. But we don't want to delay the write while
  337. * we wait for buckets to be freed up, so just invalidate the
  338. * rest of the write.
  339. */
  340. op->bypass = true;
  341. return bch_data_invalidate(cl);
  342. } else {
  343. /*
  344. * From a cache miss, we can just insert the keys for the data
  345. * we have written or bail out if we didn't do anything.
  346. */
  347. op->insert_data_done = true;
  348. bio_put(bio);
  349. if (!bch_keylist_empty(&op->insert_keys))
  350. continue_at(cl, bch_data_insert_keys, bcache_wq);
  351. else
  352. closure_return(cl);
  353. }
  354. }
  355. /**
  356. * bch_data_insert - stick some data in the cache
  357. *
  358. * This is the starting point for any data to end up in a cache device; it could
  359. * be from a normal write, or a writeback write, or a write to a flash only
  360. * volume - it's also used by the moving garbage collector to compact data in
  361. * mostly empty buckets.
  362. *
  363. * It first writes the data to the cache, creating a list of keys to be inserted
  364. * (if the data had to be fragmented there will be multiple keys); after the
  365. * data is written it calls bch_journal, and after the keys have been added to
  366. * the next journal write they're inserted into the btree.
  367. *
  368. * It inserts the data in s->cache_bio; bi_sector is used for the key offset,
  369. * and op->inode is used for the key inode.
  370. *
  371. * If s->bypass is true, instead of inserting the data it invalidates the
  372. * region of the cache represented by s->cache_bio and op->inode.
  373. */
  374. void bch_data_insert(struct closure *cl)
  375. {
  376. struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
  377. trace_bcache_write(op->bio, op->writeback, op->bypass);
  378. bch_keylist_init(&op->insert_keys);
  379. bio_get(op->bio);
  380. bch_data_insert_start(cl);
  381. }
  382. /* Congested? */
  383. unsigned bch_get_congested(struct cache_set *c)
  384. {
  385. int i;
  386. long rand;
  387. if (!c->congested_read_threshold_us &&
  388. !c->congested_write_threshold_us)
  389. return 0;
  390. i = (local_clock_us() - c->congested_last_us) / 1024;
  391. if (i < 0)
  392. return 0;
  393. i += atomic_read(&c->congested);
  394. if (i >= 0)
  395. return 0;
  396. i += CONGESTED_MAX;
  397. if (i > 0)
  398. i = fract_exp_two(i, 6);
  399. rand = get_random_int();
  400. i -= bitmap_weight(&rand, BITS_PER_LONG);
  401. return i > 0 ? i : 1;
  402. }
  403. static void add_sequential(struct task_struct *t)
  404. {
  405. ewma_add(t->sequential_io_avg,
  406. t->sequential_io, 8, 0);
  407. t->sequential_io = 0;
  408. }
  409. static struct hlist_head *iohash(struct cached_dev *dc, uint64_t k)
  410. {
  411. return &dc->io_hash[hash_64(k, RECENT_IO_BITS)];
  412. }
  413. static bool check_should_bypass(struct cached_dev *dc, struct bio *bio)
  414. {
  415. struct cache_set *c = dc->disk.c;
  416. unsigned mode = cache_mode(dc, bio);
  417. unsigned sectors, congested = bch_get_congested(c);
  418. struct task_struct *task = current;
  419. struct io *i;
  420. if (test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) ||
  421. c->gc_stats.in_use > CUTOFF_CACHE_ADD ||
  422. (bio->bi_rw & REQ_DISCARD))
  423. goto skip;
  424. if (mode == CACHE_MODE_NONE ||
  425. (mode == CACHE_MODE_WRITEAROUND &&
  426. (bio->bi_rw & REQ_WRITE)))
  427. goto skip;
  428. if (bio->bi_sector & (c->sb.block_size - 1) ||
  429. bio_sectors(bio) & (c->sb.block_size - 1)) {
  430. pr_debug("skipping unaligned io");
  431. goto skip;
  432. }
  433. if (bypass_torture_test(dc)) {
  434. if ((get_random_int() & 3) == 3)
  435. goto skip;
  436. else
  437. goto rescale;
  438. }
  439. if (!congested && !dc->sequential_cutoff)
  440. goto rescale;
  441. if (!congested &&
  442. mode == CACHE_MODE_WRITEBACK &&
  443. (bio->bi_rw & REQ_WRITE) &&
  444. (bio->bi_rw & REQ_SYNC))
  445. goto rescale;
  446. spin_lock(&dc->io_lock);
  447. hlist_for_each_entry(i, iohash(dc, bio->bi_sector), hash)
  448. if (i->last == bio->bi_sector &&
  449. time_before(jiffies, i->jiffies))
  450. goto found;
  451. i = list_first_entry(&dc->io_lru, struct io, lru);
  452. add_sequential(task);
  453. i->sequential = 0;
  454. found:
  455. if (i->sequential + bio->bi_size > i->sequential)
  456. i->sequential += bio->bi_size;
  457. i->last = bio_end_sector(bio);
  458. i->jiffies = jiffies + msecs_to_jiffies(5000);
  459. task->sequential_io = i->sequential;
  460. hlist_del(&i->hash);
  461. hlist_add_head(&i->hash, iohash(dc, i->last));
  462. list_move_tail(&i->lru, &dc->io_lru);
  463. spin_unlock(&dc->io_lock);
  464. sectors = max(task->sequential_io,
  465. task->sequential_io_avg) >> 9;
  466. if (dc->sequential_cutoff &&
  467. sectors >= dc->sequential_cutoff >> 9) {
  468. trace_bcache_bypass_sequential(bio);
  469. goto skip;
  470. }
  471. if (congested && sectors >= congested) {
  472. trace_bcache_bypass_congested(bio);
  473. goto skip;
  474. }
  475. rescale:
  476. bch_rescale_priorities(c, bio_sectors(bio));
  477. return false;
  478. skip:
  479. bch_mark_sectors_bypassed(c, dc, bio_sectors(bio));
  480. return true;
  481. }
  482. /* Cache lookup */
  483. struct search {
  484. /* Stack frame for bio_complete */
  485. struct closure cl;
  486. struct bcache_device *d;
  487. struct bbio bio;
  488. struct bio *orig_bio;
  489. struct bio *cache_miss;
  490. unsigned insert_bio_sectors;
  491. unsigned recoverable:1;
  492. unsigned unaligned_bvec:1;
  493. unsigned write:1;
  494. unsigned read_dirty_data:1;
  495. unsigned long start_time;
  496. struct btree_op op;
  497. struct data_insert_op iop;
  498. };
  499. static void bch_cache_read_endio(struct bio *bio, int error)
  500. {
  501. struct bbio *b = container_of(bio, struct bbio, bio);
  502. struct closure *cl = bio->bi_private;
  503. struct search *s = container_of(cl, struct search, cl);
  504. /*
  505. * If the bucket was reused while our bio was in flight, we might have
  506. * read the wrong data. Set s->error but not error so it doesn't get
  507. * counted against the cache device, but we'll still reread the data
  508. * from the backing device.
  509. */
  510. if (error)
  511. s->iop.error = error;
  512. else if (ptr_stale(s->iop.c, &b->key, 0)) {
  513. atomic_long_inc(&s->iop.c->cache_read_races);
  514. s->iop.error = -EINTR;
  515. }
  516. bch_bbio_endio(s->iop.c, bio, error, "reading from cache");
  517. }
  518. /*
  519. * Read from a single key, handling the initial cache miss if the key starts in
  520. * the middle of the bio
  521. */
  522. static int cache_lookup_fn(struct btree_op *op, struct btree *b, struct bkey *k)
  523. {
  524. struct search *s = container_of(op, struct search, op);
  525. struct bio *n, *bio = &s->bio.bio;
  526. struct bkey *bio_key;
  527. unsigned ptr;
  528. if (bkey_cmp(k, &KEY(s->iop.inode, bio->bi_sector, 0)) <= 0)
  529. return MAP_CONTINUE;
  530. if (KEY_INODE(k) != s->iop.inode ||
  531. KEY_START(k) > bio->bi_sector) {
  532. unsigned bio_sectors = bio_sectors(bio);
  533. unsigned sectors = KEY_INODE(k) == s->iop.inode
  534. ? min_t(uint64_t, INT_MAX,
  535. KEY_START(k) - bio->bi_sector)
  536. : INT_MAX;
  537. int ret = s->d->cache_miss(b, s, bio, sectors);
  538. if (ret != MAP_CONTINUE)
  539. return ret;
  540. /* if this was a complete miss we shouldn't get here */
  541. BUG_ON(bio_sectors <= sectors);
  542. }
  543. if (!KEY_SIZE(k))
  544. return MAP_CONTINUE;
  545. /* XXX: figure out best pointer - for multiple cache devices */
  546. ptr = 0;
  547. PTR_BUCKET(b->c, k, ptr)->prio = INITIAL_PRIO;
  548. if (KEY_DIRTY(k))
  549. s->read_dirty_data = true;
  550. n = bch_bio_split(bio, min_t(uint64_t, INT_MAX,
  551. KEY_OFFSET(k) - bio->bi_sector),
  552. GFP_NOIO, s->d->bio_split);
  553. bio_key = &container_of(n, struct bbio, bio)->key;
  554. bch_bkey_copy_single_ptr(bio_key, k, ptr);
  555. bch_cut_front(&KEY(s->iop.inode, n->bi_sector, 0), bio_key);
  556. bch_cut_back(&KEY(s->iop.inode, bio_end_sector(n), 0), bio_key);
  557. n->bi_end_io = bch_cache_read_endio;
  558. n->bi_private = &s->cl;
  559. /*
  560. * The bucket we're reading from might be reused while our bio
  561. * is in flight, and we could then end up reading the wrong
  562. * data.
  563. *
  564. * We guard against this by checking (in cache_read_endio()) if
  565. * the pointer is stale again; if so, we treat it as an error
  566. * and reread from the backing device (but we don't pass that
  567. * error up anywhere).
  568. */
  569. __bch_submit_bbio(n, b->c);
  570. return n == bio ? MAP_DONE : MAP_CONTINUE;
  571. }
  572. static void cache_lookup(struct closure *cl)
  573. {
  574. struct search *s = container_of(cl, struct search, iop.cl);
  575. struct bio *bio = &s->bio.bio;
  576. int ret = bch_btree_map_keys(&s->op, s->iop.c,
  577. &KEY(s->iop.inode, bio->bi_sector, 0),
  578. cache_lookup_fn, MAP_END_KEY);
  579. if (ret == -EAGAIN)
  580. continue_at(cl, cache_lookup, bcache_wq);
  581. closure_return(cl);
  582. }
  583. /* Common code for the make_request functions */
  584. static void request_endio(struct bio *bio, int error)
  585. {
  586. struct closure *cl = bio->bi_private;
  587. if (error) {
  588. struct search *s = container_of(cl, struct search, cl);
  589. s->iop.error = error;
  590. /* Only cache read errors are recoverable */
  591. s->recoverable = false;
  592. }
  593. bio_put(bio);
  594. closure_put(cl);
  595. }
  596. static void bio_complete(struct search *s)
  597. {
  598. if (s->orig_bio) {
  599. int cpu, rw = bio_data_dir(s->orig_bio);
  600. unsigned long duration = jiffies - s->start_time;
  601. cpu = part_stat_lock();
  602. part_round_stats(cpu, &s->d->disk->part0);
  603. part_stat_add(cpu, &s->d->disk->part0, ticks[rw], duration);
  604. part_stat_unlock();
  605. trace_bcache_request_end(s->d, s->orig_bio);
  606. bio_endio(s->orig_bio, s->iop.error);
  607. s->orig_bio = NULL;
  608. }
  609. }
  610. static void do_bio_hook(struct search *s)
  611. {
  612. struct bio *bio = &s->bio.bio;
  613. memcpy(bio, s->orig_bio, sizeof(struct bio));
  614. bio->bi_end_io = request_endio;
  615. bio->bi_private = &s->cl;
  616. atomic_set(&bio->bi_cnt, 3);
  617. }
  618. static void search_free(struct closure *cl)
  619. {
  620. struct search *s = container_of(cl, struct search, cl);
  621. bio_complete(s);
  622. if (s->iop.bio)
  623. bio_put(s->iop.bio);
  624. if (s->unaligned_bvec)
  625. mempool_free(s->bio.bio.bi_io_vec, s->d->unaligned_bvec);
  626. closure_debug_destroy(cl);
  627. mempool_free(s, s->d->c->search);
  628. }
  629. static struct search *search_alloc(struct bio *bio, struct bcache_device *d)
  630. {
  631. struct search *s;
  632. struct bio_vec *bv;
  633. s = mempool_alloc(d->c->search, GFP_NOIO);
  634. memset(s, 0, offsetof(struct search, iop.insert_keys));
  635. __closure_init(&s->cl, NULL);
  636. s->iop.inode = d->id;
  637. s->iop.c = d->c;
  638. s->d = d;
  639. s->op.lock = -1;
  640. s->iop.write_point = hash_long((unsigned long) current, 16);
  641. s->orig_bio = bio;
  642. s->write = (bio->bi_rw & REQ_WRITE) != 0;
  643. s->iop.flush_journal = (bio->bi_rw & (REQ_FLUSH|REQ_FUA)) != 0;
  644. s->recoverable = 1;
  645. s->start_time = jiffies;
  646. do_bio_hook(s);
  647. if (bio->bi_size != bio_segments(bio) * PAGE_SIZE) {
  648. bv = mempool_alloc(d->unaligned_bvec, GFP_NOIO);
  649. memcpy(bv, bio_iovec(bio),
  650. sizeof(struct bio_vec) * bio_segments(bio));
  651. s->bio.bio.bi_io_vec = bv;
  652. s->unaligned_bvec = 1;
  653. }
  654. return s;
  655. }
  656. /* Cached devices */
  657. static void cached_dev_bio_complete(struct closure *cl)
  658. {
  659. struct search *s = container_of(cl, struct search, cl);
  660. struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
  661. search_free(cl);
  662. cached_dev_put(dc);
  663. }
  664. /* Process reads */
  665. static void cached_dev_cache_miss_done(struct closure *cl)
  666. {
  667. struct search *s = container_of(cl, struct search, cl);
  668. if (s->iop.replace_collision)
  669. bch_mark_cache_miss_collision(s->iop.c, s->d);
  670. if (s->iop.bio) {
  671. int i;
  672. struct bio_vec *bv;
  673. bio_for_each_segment_all(bv, s->iop.bio, i)
  674. __free_page(bv->bv_page);
  675. }
  676. cached_dev_bio_complete(cl);
  677. }
  678. static void cached_dev_read_error(struct closure *cl)
  679. {
  680. struct search *s = container_of(cl, struct search, cl);
  681. struct bio *bio = &s->bio.bio;
  682. struct bio_vec *bv;
  683. int i;
  684. if (s->recoverable) {
  685. /* Retry from the backing device: */
  686. trace_bcache_read_retry(s->orig_bio);
  687. s->iop.error = 0;
  688. bv = s->bio.bio.bi_io_vec;
  689. do_bio_hook(s);
  690. s->bio.bio.bi_io_vec = bv;
  691. if (!s->unaligned_bvec)
  692. bio_for_each_segment(bv, s->orig_bio, i)
  693. bv->bv_offset = 0, bv->bv_len = PAGE_SIZE;
  694. else
  695. memcpy(s->bio.bio.bi_io_vec,
  696. bio_iovec(s->orig_bio),
  697. sizeof(struct bio_vec) *
  698. bio_segments(s->orig_bio));
  699. /* XXX: invalidate cache */
  700. closure_bio_submit(bio, cl, s->d);
  701. }
  702. continue_at(cl, cached_dev_cache_miss_done, NULL);
  703. }
  704. static void cached_dev_read_done(struct closure *cl)
  705. {
  706. struct search *s = container_of(cl, struct search, cl);
  707. struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
  708. /*
  709. * We had a cache miss; cache_bio now contains data ready to be inserted
  710. * into the cache.
  711. *
  712. * First, we copy the data we just read from cache_bio's bounce buffers
  713. * to the buffers the original bio pointed to:
  714. */
  715. if (s->iop.bio) {
  716. bio_reset(s->iop.bio);
  717. s->iop.bio->bi_sector = s->cache_miss->bi_sector;
  718. s->iop.bio->bi_bdev = s->cache_miss->bi_bdev;
  719. s->iop.bio->bi_size = s->insert_bio_sectors << 9;
  720. bch_bio_map(s->iop.bio, NULL);
  721. bio_copy_data(s->cache_miss, s->iop.bio);
  722. bio_put(s->cache_miss);
  723. s->cache_miss = NULL;
  724. }
  725. if (verify(dc, &s->bio.bio) && s->recoverable &&
  726. !s->unaligned_bvec && !s->read_dirty_data)
  727. bch_data_verify(dc, s->orig_bio);
  728. bio_complete(s);
  729. if (s->iop.bio &&
  730. !test_bit(CACHE_SET_STOPPING, &s->iop.c->flags)) {
  731. BUG_ON(!s->iop.replace);
  732. closure_call(&s->iop.cl, bch_data_insert, NULL, cl);
  733. }
  734. continue_at(cl, cached_dev_cache_miss_done, NULL);
  735. }
  736. static void cached_dev_read_done_bh(struct closure *cl)
  737. {
  738. struct search *s = container_of(cl, struct search, cl);
  739. struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
  740. bch_mark_cache_accounting(s->iop.c, s->d,
  741. !s->cache_miss, s->iop.bypass);
  742. trace_bcache_read(s->orig_bio, !s->cache_miss, s->iop.bypass);
  743. if (s->iop.error)
  744. continue_at_nobarrier(cl, cached_dev_read_error, bcache_wq);
  745. else if (s->iop.bio || verify(dc, &s->bio.bio))
  746. continue_at_nobarrier(cl, cached_dev_read_done, bcache_wq);
  747. else
  748. continue_at_nobarrier(cl, cached_dev_bio_complete, NULL);
  749. }
  750. static int cached_dev_cache_miss(struct btree *b, struct search *s,
  751. struct bio *bio, unsigned sectors)
  752. {
  753. int ret = MAP_CONTINUE;
  754. unsigned reada = 0;
  755. struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
  756. struct bio *miss, *cache_bio;
  757. if (s->cache_miss || s->iop.bypass) {
  758. miss = bch_bio_split(bio, sectors, GFP_NOIO, s->d->bio_split);
  759. ret = miss == bio ? MAP_DONE : MAP_CONTINUE;
  760. goto out_submit;
  761. }
  762. if (!(bio->bi_rw & REQ_RAHEAD) &&
  763. !(bio->bi_rw & REQ_META) &&
  764. s->iop.c->gc_stats.in_use < CUTOFF_CACHE_READA)
  765. reada = min_t(sector_t, dc->readahead >> 9,
  766. bdev_sectors(bio->bi_bdev) - bio_end_sector(bio));
  767. s->insert_bio_sectors = min(sectors, bio_sectors(bio) + reada);
  768. s->iop.replace_key = KEY(s->iop.inode,
  769. bio->bi_sector + s->insert_bio_sectors,
  770. s->insert_bio_sectors);
  771. ret = bch_btree_insert_check_key(b, &s->op, &s->iop.replace_key);
  772. if (ret)
  773. return ret;
  774. s->iop.replace = true;
  775. miss = bch_bio_split(bio, sectors, GFP_NOIO, s->d->bio_split);
  776. /* btree_search_recurse()'s btree iterator is no good anymore */
  777. ret = miss == bio ? MAP_DONE : -EINTR;
  778. cache_bio = bio_alloc_bioset(GFP_NOWAIT,
  779. DIV_ROUND_UP(s->insert_bio_sectors, PAGE_SECTORS),
  780. dc->disk.bio_split);
  781. if (!cache_bio)
  782. goto out_submit;
  783. cache_bio->bi_sector = miss->bi_sector;
  784. cache_bio->bi_bdev = miss->bi_bdev;
  785. cache_bio->bi_size = s->insert_bio_sectors << 9;
  786. cache_bio->bi_end_io = request_endio;
  787. cache_bio->bi_private = &s->cl;
  788. bch_bio_map(cache_bio, NULL);
  789. if (bio_alloc_pages(cache_bio, __GFP_NOWARN|GFP_NOIO))
  790. goto out_put;
  791. if (reada)
  792. bch_mark_cache_readahead(s->iop.c, s->d);
  793. s->cache_miss = miss;
  794. s->iop.bio = cache_bio;
  795. bio_get(cache_bio);
  796. closure_bio_submit(cache_bio, &s->cl, s->d);
  797. return ret;
  798. out_put:
  799. bio_put(cache_bio);
  800. out_submit:
  801. miss->bi_end_io = request_endio;
  802. miss->bi_private = &s->cl;
  803. closure_bio_submit(miss, &s->cl, s->d);
  804. return ret;
  805. }
  806. static void cached_dev_read(struct cached_dev *dc, struct search *s)
  807. {
  808. struct closure *cl = &s->cl;
  809. closure_call(&s->iop.cl, cache_lookup, NULL, cl);
  810. continue_at(cl, cached_dev_read_done_bh, NULL);
  811. }
  812. /* Process writes */
  813. static void cached_dev_write_complete(struct closure *cl)
  814. {
  815. struct search *s = container_of(cl, struct search, cl);
  816. struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
  817. up_read_non_owner(&dc->writeback_lock);
  818. cached_dev_bio_complete(cl);
  819. }
  820. static void cached_dev_write(struct cached_dev *dc, struct search *s)
  821. {
  822. struct closure *cl = &s->cl;
  823. struct bio *bio = &s->bio.bio;
  824. struct bkey start = KEY(dc->disk.id, bio->bi_sector, 0);
  825. struct bkey end = KEY(dc->disk.id, bio_end_sector(bio), 0);
  826. bch_keybuf_check_overlapping(&s->iop.c->moving_gc_keys, &start, &end);
  827. down_read_non_owner(&dc->writeback_lock);
  828. if (bch_keybuf_check_overlapping(&dc->writeback_keys, &start, &end)) {
  829. /*
  830. * We overlap with some dirty data undergoing background
  831. * writeback, force this write to writeback
  832. */
  833. s->iop.bypass = false;
  834. s->iop.writeback = true;
  835. }
  836. /*
  837. * Discards aren't _required_ to do anything, so skipping if
  838. * check_overlapping returned true is ok
  839. *
  840. * But check_overlapping drops dirty keys for which io hasn't started,
  841. * so we still want to call it.
  842. */
  843. if (bio->bi_rw & REQ_DISCARD)
  844. s->iop.bypass = true;
  845. if (should_writeback(dc, s->orig_bio,
  846. cache_mode(dc, bio),
  847. s->iop.bypass)) {
  848. s->iop.bypass = false;
  849. s->iop.writeback = true;
  850. }
  851. if (s->iop.bypass) {
  852. s->iop.bio = s->orig_bio;
  853. bio_get(s->iop.bio);
  854. if (!(bio->bi_rw & REQ_DISCARD) ||
  855. blk_queue_discard(bdev_get_queue(dc->bdev)))
  856. closure_bio_submit(bio, cl, s->d);
  857. } else if (s->iop.writeback) {
  858. bch_writeback_add(dc);
  859. s->iop.bio = bio;
  860. if (bio->bi_rw & REQ_FLUSH) {
  861. /* Also need to send a flush to the backing device */
  862. struct bio *flush = bio_alloc_bioset(GFP_NOIO, 0,
  863. dc->disk.bio_split);
  864. flush->bi_rw = WRITE_FLUSH;
  865. flush->bi_bdev = bio->bi_bdev;
  866. flush->bi_end_io = request_endio;
  867. flush->bi_private = cl;
  868. closure_bio_submit(flush, cl, s->d);
  869. }
  870. } else {
  871. s->iop.bio = bio_clone_bioset(bio, GFP_NOIO,
  872. dc->disk.bio_split);
  873. closure_bio_submit(bio, cl, s->d);
  874. }
  875. closure_call(&s->iop.cl, bch_data_insert, NULL, cl);
  876. continue_at(cl, cached_dev_write_complete, NULL);
  877. }
  878. static void cached_dev_nodata(struct closure *cl)
  879. {
  880. struct search *s = container_of(cl, struct search, cl);
  881. struct bio *bio = &s->bio.bio;
  882. if (s->iop.flush_journal)
  883. bch_journal_meta(s->iop.c, cl);
  884. /* If it's a flush, we send the flush to the backing device too */
  885. closure_bio_submit(bio, cl, s->d);
  886. continue_at(cl, cached_dev_bio_complete, NULL);
  887. }
  888. /* Cached devices - read & write stuff */
  889. static void cached_dev_make_request(struct request_queue *q, struct bio *bio)
  890. {
  891. struct search *s;
  892. struct bcache_device *d = bio->bi_bdev->bd_disk->private_data;
  893. struct cached_dev *dc = container_of(d, struct cached_dev, disk);
  894. int cpu, rw = bio_data_dir(bio);
  895. cpu = part_stat_lock();
  896. part_stat_inc(cpu, &d->disk->part0, ios[rw]);
  897. part_stat_add(cpu, &d->disk->part0, sectors[rw], bio_sectors(bio));
  898. part_stat_unlock();
  899. bio->bi_bdev = dc->bdev;
  900. bio->bi_sector += dc->sb.data_offset;
  901. if (cached_dev_get(dc)) {
  902. s = search_alloc(bio, d);
  903. trace_bcache_request_start(s->d, bio);
  904. if (!bio->bi_size) {
  905. /*
  906. * can't call bch_journal_meta from under
  907. * generic_make_request
  908. */
  909. continue_at_nobarrier(&s->cl,
  910. cached_dev_nodata,
  911. bcache_wq);
  912. } else {
  913. s->iop.bypass = check_should_bypass(dc, bio);
  914. if (rw)
  915. cached_dev_write(dc, s);
  916. else
  917. cached_dev_read(dc, s);
  918. }
  919. } else {
  920. if ((bio->bi_rw & REQ_DISCARD) &&
  921. !blk_queue_discard(bdev_get_queue(dc->bdev)))
  922. bio_endio(bio, 0);
  923. else
  924. bch_generic_make_request(bio, &d->bio_split_hook);
  925. }
  926. }
  927. static int cached_dev_ioctl(struct bcache_device *d, fmode_t mode,
  928. unsigned int cmd, unsigned long arg)
  929. {
  930. struct cached_dev *dc = container_of(d, struct cached_dev, disk);
  931. return __blkdev_driver_ioctl(dc->bdev, mode, cmd, arg);
  932. }
  933. static int cached_dev_congested(void *data, int bits)
  934. {
  935. struct bcache_device *d = data;
  936. struct cached_dev *dc = container_of(d, struct cached_dev, disk);
  937. struct request_queue *q = bdev_get_queue(dc->bdev);
  938. int ret = 0;
  939. if (bdi_congested(&q->backing_dev_info, bits))
  940. return 1;
  941. if (cached_dev_get(dc)) {
  942. unsigned i;
  943. struct cache *ca;
  944. for_each_cache(ca, d->c, i) {
  945. q = bdev_get_queue(ca->bdev);
  946. ret |= bdi_congested(&q->backing_dev_info, bits);
  947. }
  948. cached_dev_put(dc);
  949. }
  950. return ret;
  951. }
  952. void bch_cached_dev_request_init(struct cached_dev *dc)
  953. {
  954. struct gendisk *g = dc->disk.disk;
  955. g->queue->make_request_fn = cached_dev_make_request;
  956. g->queue->backing_dev_info.congested_fn = cached_dev_congested;
  957. dc->disk.cache_miss = cached_dev_cache_miss;
  958. dc->disk.ioctl = cached_dev_ioctl;
  959. }
  960. /* Flash backed devices */
  961. static int flash_dev_cache_miss(struct btree *b, struct search *s,
  962. struct bio *bio, unsigned sectors)
  963. {
  964. struct bio_vec *bv;
  965. int i;
  966. /* Zero fill bio */
  967. bio_for_each_segment(bv, bio, i) {
  968. unsigned j = min(bv->bv_len >> 9, sectors);
  969. void *p = kmap(bv->bv_page);
  970. memset(p + bv->bv_offset, 0, j << 9);
  971. kunmap(bv->bv_page);
  972. sectors -= j;
  973. }
  974. bio_advance(bio, min(sectors << 9, bio->bi_size));
  975. if (!bio->bi_size)
  976. return MAP_DONE;
  977. return MAP_CONTINUE;
  978. }
  979. static void flash_dev_nodata(struct closure *cl)
  980. {
  981. struct search *s = container_of(cl, struct search, cl);
  982. if (s->iop.flush_journal)
  983. bch_journal_meta(s->iop.c, cl);
  984. continue_at(cl, search_free, NULL);
  985. }
  986. static void flash_dev_make_request(struct request_queue *q, struct bio *bio)
  987. {
  988. struct search *s;
  989. struct closure *cl;
  990. struct bcache_device *d = bio->bi_bdev->bd_disk->private_data;
  991. int cpu, rw = bio_data_dir(bio);
  992. cpu = part_stat_lock();
  993. part_stat_inc(cpu, &d->disk->part0, ios[rw]);
  994. part_stat_add(cpu, &d->disk->part0, sectors[rw], bio_sectors(bio));
  995. part_stat_unlock();
  996. s = search_alloc(bio, d);
  997. cl = &s->cl;
  998. bio = &s->bio.bio;
  999. trace_bcache_request_start(s->d, bio);
  1000. if (!bio->bi_size) {
  1001. /*
  1002. * can't call bch_journal_meta from under
  1003. * generic_make_request
  1004. */
  1005. continue_at_nobarrier(&s->cl,
  1006. flash_dev_nodata,
  1007. bcache_wq);
  1008. } else if (rw) {
  1009. bch_keybuf_check_overlapping(&s->iop.c->moving_gc_keys,
  1010. &KEY(d->id, bio->bi_sector, 0),
  1011. &KEY(d->id, bio_end_sector(bio), 0));
  1012. s->iop.bypass = (bio->bi_rw & REQ_DISCARD) != 0;
  1013. s->iop.writeback = true;
  1014. s->iop.bio = bio;
  1015. closure_call(&s->iop.cl, bch_data_insert, NULL, cl);
  1016. } else {
  1017. closure_call(&s->iop.cl, cache_lookup, NULL, cl);
  1018. }
  1019. continue_at(cl, search_free, NULL);
  1020. }
  1021. static int flash_dev_ioctl(struct bcache_device *d, fmode_t mode,
  1022. unsigned int cmd, unsigned long arg)
  1023. {
  1024. return -ENOTTY;
  1025. }
  1026. static int flash_dev_congested(void *data, int bits)
  1027. {
  1028. struct bcache_device *d = data;
  1029. struct request_queue *q;
  1030. struct cache *ca;
  1031. unsigned i;
  1032. int ret = 0;
  1033. for_each_cache(ca, d->c, i) {
  1034. q = bdev_get_queue(ca->bdev);
  1035. ret |= bdi_congested(&q->backing_dev_info, bits);
  1036. }
  1037. return ret;
  1038. }
  1039. void bch_flash_dev_request_init(struct bcache_device *d)
  1040. {
  1041. struct gendisk *g = d->disk;
  1042. g->queue->make_request_fn = flash_dev_make_request;
  1043. g->queue->backing_dev_info.congested_fn = flash_dev_congested;
  1044. d->cache_miss = flash_dev_cache_miss;
  1045. d->ioctl = flash_dev_ioctl;
  1046. }
  1047. void bch_request_exit(void)
  1048. {
  1049. #ifdef CONFIG_CGROUP_BCACHE
  1050. cgroup_unload_subsys(&bcache_subsys);
  1051. #endif
  1052. if (bch_search_cache)
  1053. kmem_cache_destroy(bch_search_cache);
  1054. }
  1055. int __init bch_request_init(void)
  1056. {
  1057. bch_search_cache = KMEM_CACHE(search, 0);
  1058. if (!bch_search_cache)
  1059. return -ENOMEM;
  1060. #ifdef CONFIG_CGROUP_BCACHE
  1061. cgroup_load_subsys(&bcache_subsys);
  1062. init_bch_cgroup(&bcache_default_cgroup);
  1063. cgroup_add_cftypes(&bcache_subsys, bch_files);
  1064. #endif
  1065. return 0;
  1066. }