bio-integrity.c 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750
  1. /*
  2. * bio-integrity.c - bio data integrity extensions
  3. *
  4. * Copyright (C) 2007, 2008 Oracle Corporation
  5. * Written by: Martin K. Petersen <martin.petersen@oracle.com>
  6. *
  7. * This program is free software; you can redistribute it and/or
  8. * modify it under the terms of the GNU General Public License version
  9. * 2 as published by the Free Software Foundation.
  10. *
  11. * This program is distributed in the hope that it will be useful, but
  12. * WITHOUT ANY WARRANTY; without even the implied warranty of
  13. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14. * General Public License for more details.
  15. *
  16. * You should have received a copy of the GNU General Public License
  17. * along with this program; see the file COPYING. If not, write to
  18. * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
  19. * USA.
  20. *
  21. */
  22. #include <linux/blkdev.h>
  23. #include <linux/mempool.h>
  24. #include <linux/bio.h>
  25. #include <linux/workqueue.h>
  26. static struct kmem_cache *bio_integrity_slab __read_mostly;
  27. static struct workqueue_struct *kintegrityd_wq;
  28. /**
  29. * bio_integrity_alloc_bioset - Allocate integrity payload and attach it to bio
  30. * @bio: bio to attach integrity metadata to
  31. * @gfp_mask: Memory allocation mask
  32. * @nr_vecs: Number of integrity metadata scatter-gather elements
  33. * @bs: bio_set to allocate from
  34. *
  35. * Description: This function prepares a bio for attaching integrity
  36. * metadata. nr_vecs specifies the maximum number of pages containing
  37. * integrity metadata that can be attached.
  38. */
  39. struct bio_integrity_payload *bio_integrity_alloc_bioset(struct bio *bio,
  40. gfp_t gfp_mask,
  41. unsigned int nr_vecs,
  42. struct bio_set *bs)
  43. {
  44. struct bio_integrity_payload *bip;
  45. struct bio_vec *iv;
  46. unsigned long idx;
  47. BUG_ON(bio == NULL);
  48. bip = mempool_alloc(bs->bio_integrity_pool, gfp_mask);
  49. if (unlikely(bip == NULL)) {
  50. printk(KERN_ERR "%s: could not alloc bip\n", __func__);
  51. return NULL;
  52. }
  53. memset(bip, 0, sizeof(*bip));
  54. iv = bvec_alloc_bs(gfp_mask, nr_vecs, &idx, bs);
  55. if (unlikely(iv == NULL)) {
  56. printk(KERN_ERR "%s: could not alloc bip_vec\n", __func__);
  57. mempool_free(bip, bs->bio_integrity_pool);
  58. return NULL;
  59. }
  60. bip->bip_pool = idx;
  61. bip->bip_vec = iv;
  62. bip->bip_bio = bio;
  63. bio->bi_integrity = bip;
  64. return bip;
  65. }
  66. EXPORT_SYMBOL(bio_integrity_alloc_bioset);
  67. /**
  68. * bio_integrity_alloc - Allocate integrity payload and attach it to bio
  69. * @bio: bio to attach integrity metadata to
  70. * @gfp_mask: Memory allocation mask
  71. * @nr_vecs: Number of integrity metadata scatter-gather elements
  72. *
  73. * Description: This function prepares a bio for attaching integrity
  74. * metadata. nr_vecs specifies the maximum number of pages containing
  75. * integrity metadata that can be attached.
  76. */
  77. struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,
  78. gfp_t gfp_mask,
  79. unsigned int nr_vecs)
  80. {
  81. return bio_integrity_alloc_bioset(bio, gfp_mask, nr_vecs, fs_bio_set);
  82. }
  83. EXPORT_SYMBOL(bio_integrity_alloc);
  84. /**
  85. * bio_integrity_free - Free bio integrity payload
  86. * @bio: bio containing bip to be freed
  87. * @bs: bio_set this bio was allocated from
  88. *
  89. * Description: Used to free the integrity portion of a bio. Usually
  90. * called from bio_free().
  91. */
  92. void bio_integrity_free(struct bio *bio, struct bio_set *bs)
  93. {
  94. struct bio_integrity_payload *bip = bio->bi_integrity;
  95. BUG_ON(bip == NULL);
  96. /* A cloned bio doesn't own the integrity metadata */
  97. if (!bio_flagged(bio, BIO_CLONED) && !bio_flagged(bio, BIO_FS_INTEGRITY)
  98. && bip->bip_buf != NULL)
  99. kfree(bip->bip_buf);
  100. bvec_free_bs(bs, bip->bip_vec, bip->bip_pool);
  101. mempool_free(bip, bs->bio_integrity_pool);
  102. bio->bi_integrity = NULL;
  103. }
  104. EXPORT_SYMBOL(bio_integrity_free);
  105. /**
  106. * bio_integrity_add_page - Attach integrity metadata
  107. * @bio: bio to update
  108. * @page: page containing integrity metadata
  109. * @len: number of bytes of integrity metadata in page
  110. * @offset: start offset within page
  111. *
  112. * Description: Attach a page containing integrity metadata to bio.
  113. */
  114. int bio_integrity_add_page(struct bio *bio, struct page *page,
  115. unsigned int len, unsigned int offset)
  116. {
  117. struct bio_integrity_payload *bip = bio->bi_integrity;
  118. struct bio_vec *iv;
  119. if (bip->bip_vcnt >= bvec_nr_vecs(bip->bip_pool)) {
  120. printk(KERN_ERR "%s: bip_vec full\n", __func__);
  121. return 0;
  122. }
  123. iv = bip_vec_idx(bip, bip->bip_vcnt);
  124. BUG_ON(iv == NULL);
  125. BUG_ON(iv->bv_page != NULL);
  126. iv->bv_page = page;
  127. iv->bv_len = len;
  128. iv->bv_offset = offset;
  129. bip->bip_vcnt++;
  130. return len;
  131. }
  132. EXPORT_SYMBOL(bio_integrity_add_page);
  133. static int bdev_integrity_enabled(struct block_device *bdev, int rw)
  134. {
  135. struct blk_integrity *bi = bdev_get_integrity(bdev);
  136. if (bi == NULL)
  137. return 0;
  138. if (rw == READ && bi->verify_fn != NULL &&
  139. (bi->flags & INTEGRITY_FLAG_READ))
  140. return 1;
  141. if (rw == WRITE && bi->generate_fn != NULL &&
  142. (bi->flags & INTEGRITY_FLAG_WRITE))
  143. return 1;
  144. return 0;
  145. }
  146. /**
  147. * bio_integrity_enabled - Check whether integrity can be passed
  148. * @bio: bio to check
  149. *
  150. * Description: Determines whether bio_integrity_prep() can be called
  151. * on this bio or not. bio data direction and target device must be
  152. * set prior to calling. The functions honors the write_generate and
  153. * read_verify flags in sysfs.
  154. */
  155. int bio_integrity_enabled(struct bio *bio)
  156. {
  157. /* Already protected? */
  158. if (bio_integrity(bio))
  159. return 0;
  160. return bdev_integrity_enabled(bio->bi_bdev, bio_data_dir(bio));
  161. }
  162. EXPORT_SYMBOL(bio_integrity_enabled);
  163. /**
  164. * bio_integrity_hw_sectors - Convert 512b sectors to hardware ditto
  165. * @bi: blk_integrity profile for device
  166. * @sectors: Number of 512 sectors to convert
  167. *
  168. * Description: The block layer calculates everything in 512 byte
  169. * sectors but integrity metadata is done in terms of the hardware
  170. * sector size of the storage device. Convert the block layer sectors
  171. * to physical sectors.
  172. */
  173. static inline unsigned int bio_integrity_hw_sectors(struct blk_integrity *bi,
  174. unsigned int sectors)
  175. {
  176. /* At this point there are only 512b or 4096b DIF/EPP devices */
  177. if (bi->sector_size == 4096)
  178. return sectors >>= 3;
  179. return sectors;
  180. }
  181. /**
  182. * bio_integrity_tag_size - Retrieve integrity tag space
  183. * @bio: bio to inspect
  184. *
  185. * Description: Returns the maximum number of tag bytes that can be
  186. * attached to this bio. Filesystems can use this to determine how
  187. * much metadata to attach to an I/O.
  188. */
  189. unsigned int bio_integrity_tag_size(struct bio *bio)
  190. {
  191. struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
  192. BUG_ON(bio->bi_size == 0);
  193. return bi->tag_size * (bio->bi_size / bi->sector_size);
  194. }
  195. EXPORT_SYMBOL(bio_integrity_tag_size);
  196. int bio_integrity_tag(struct bio *bio, void *tag_buf, unsigned int len, int set)
  197. {
  198. struct bio_integrity_payload *bip = bio->bi_integrity;
  199. struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
  200. unsigned int nr_sectors;
  201. BUG_ON(bip->bip_buf == NULL);
  202. if (bi->tag_size == 0)
  203. return -1;
  204. nr_sectors = bio_integrity_hw_sectors(bi,
  205. DIV_ROUND_UP(len, bi->tag_size));
  206. if (nr_sectors * bi->tuple_size > bip->bip_size) {
  207. printk(KERN_ERR "%s: tag too big for bio: %u > %u\n",
  208. __func__, nr_sectors * bi->tuple_size, bip->bip_size);
  209. return -1;
  210. }
  211. if (set)
  212. bi->set_tag_fn(bip->bip_buf, tag_buf, nr_sectors);
  213. else
  214. bi->get_tag_fn(bip->bip_buf, tag_buf, nr_sectors);
  215. return 0;
  216. }
  217. /**
  218. * bio_integrity_set_tag - Attach a tag buffer to a bio
  219. * @bio: bio to attach buffer to
  220. * @tag_buf: Pointer to a buffer containing tag data
  221. * @len: Length of the included buffer
  222. *
  223. * Description: Use this function to tag a bio by leveraging the extra
  224. * space provided by devices formatted with integrity protection. The
  225. * size of the integrity buffer must be <= to the size reported by
  226. * bio_integrity_tag_size().
  227. */
  228. int bio_integrity_set_tag(struct bio *bio, void *tag_buf, unsigned int len)
  229. {
  230. BUG_ON(bio_data_dir(bio) != WRITE);
  231. return bio_integrity_tag(bio, tag_buf, len, 1);
  232. }
  233. EXPORT_SYMBOL(bio_integrity_set_tag);
  234. /**
  235. * bio_integrity_get_tag - Retrieve a tag buffer from a bio
  236. * @bio: bio to retrieve buffer from
  237. * @tag_buf: Pointer to a buffer for the tag data
  238. * @len: Length of the target buffer
  239. *
  240. * Description: Use this function to retrieve the tag buffer from a
  241. * completed I/O. The size of the integrity buffer must be <= to the
  242. * size reported by bio_integrity_tag_size().
  243. */
  244. int bio_integrity_get_tag(struct bio *bio, void *tag_buf, unsigned int len)
  245. {
  246. BUG_ON(bio_data_dir(bio) != READ);
  247. return bio_integrity_tag(bio, tag_buf, len, 0);
  248. }
  249. EXPORT_SYMBOL(bio_integrity_get_tag);
  250. /**
  251. * bio_integrity_generate - Generate integrity metadata for a bio
  252. * @bio: bio to generate integrity metadata for
  253. *
  254. * Description: Generates integrity metadata for a bio by calling the
  255. * block device's generation callback function. The bio must have a
  256. * bip attached with enough room to accommodate the generated
  257. * integrity metadata.
  258. */
  259. static void bio_integrity_generate(struct bio *bio)
  260. {
  261. struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
  262. struct blk_integrity_exchg bix;
  263. struct bio_vec *bv;
  264. sector_t sector = bio->bi_sector;
  265. unsigned int i, sectors, total;
  266. void *prot_buf = bio->bi_integrity->bip_buf;
  267. total = 0;
  268. bix.disk_name = bio->bi_bdev->bd_disk->disk_name;
  269. bix.sector_size = bi->sector_size;
  270. bio_for_each_segment(bv, bio, i) {
  271. void *kaddr = kmap_atomic(bv->bv_page, KM_USER0);
  272. bix.data_buf = kaddr + bv->bv_offset;
  273. bix.data_size = bv->bv_len;
  274. bix.prot_buf = prot_buf;
  275. bix.sector = sector;
  276. bi->generate_fn(&bix);
  277. sectors = bv->bv_len / bi->sector_size;
  278. sector += sectors;
  279. prot_buf += sectors * bi->tuple_size;
  280. total += sectors * bi->tuple_size;
  281. BUG_ON(total > bio->bi_integrity->bip_size);
  282. kunmap_atomic(kaddr, KM_USER0);
  283. }
  284. }
  285. static inline unsigned short blk_integrity_tuple_size(struct blk_integrity *bi)
  286. {
  287. if (bi)
  288. return bi->tuple_size;
  289. return 0;
  290. }
  291. /**
  292. * bio_integrity_prep - Prepare bio for integrity I/O
  293. * @bio: bio to prepare
  294. *
  295. * Description: Allocates a buffer for integrity metadata, maps the
  296. * pages and attaches them to a bio. The bio must have data
  297. * direction, target device and start sector set priot to calling. In
  298. * the WRITE case, integrity metadata will be generated using the
  299. * block device's integrity function. In the READ case, the buffer
  300. * will be prepared for DMA and a suitable end_io handler set up.
  301. */
  302. int bio_integrity_prep(struct bio *bio)
  303. {
  304. struct bio_integrity_payload *bip;
  305. struct blk_integrity *bi;
  306. struct request_queue *q;
  307. void *buf;
  308. unsigned long start, end;
  309. unsigned int len, nr_pages;
  310. unsigned int bytes, offset, i;
  311. unsigned int sectors;
  312. bi = bdev_get_integrity(bio->bi_bdev);
  313. q = bdev_get_queue(bio->bi_bdev);
  314. BUG_ON(bi == NULL);
  315. BUG_ON(bio_integrity(bio));
  316. sectors = bio_integrity_hw_sectors(bi, bio_sectors(bio));
  317. /* Allocate kernel buffer for protection data */
  318. len = sectors * blk_integrity_tuple_size(bi);
  319. buf = kmalloc(len, GFP_NOIO | __GFP_NOFAIL | q->bounce_gfp);
  320. if (unlikely(buf == NULL)) {
  321. printk(KERN_ERR "could not allocate integrity buffer\n");
  322. return -EIO;
  323. }
  324. end = (((unsigned long) buf) + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
  325. start = ((unsigned long) buf) >> PAGE_SHIFT;
  326. nr_pages = end - start;
  327. /* Allocate bio integrity payload and integrity vectors */
  328. bip = bio_integrity_alloc(bio, GFP_NOIO, nr_pages);
  329. if (unlikely(bip == NULL)) {
  330. printk(KERN_ERR "could not allocate data integrity bioset\n");
  331. kfree(buf);
  332. return -EIO;
  333. }
  334. bip->bip_buf = buf;
  335. bip->bip_size = len;
  336. bip->bip_sector = bio->bi_sector;
  337. /* Map it */
  338. offset = offset_in_page(buf);
  339. for (i = 0 ; i < nr_pages ; i++) {
  340. int ret;
  341. bytes = PAGE_SIZE - offset;
  342. if (len <= 0)
  343. break;
  344. if (bytes > len)
  345. bytes = len;
  346. ret = bio_integrity_add_page(bio, virt_to_page(buf),
  347. bytes, offset);
  348. if (ret == 0)
  349. return 0;
  350. if (ret < bytes)
  351. break;
  352. buf += bytes;
  353. len -= bytes;
  354. offset = 0;
  355. }
  356. /* Install custom I/O completion handler if read verify is enabled */
  357. if (bio_data_dir(bio) == READ) {
  358. bip->bip_end_io = bio->bi_end_io;
  359. bio->bi_end_io = bio_integrity_endio;
  360. }
  361. /* Auto-generate integrity metadata if this is a write */
  362. if (bio_data_dir(bio) == WRITE)
  363. bio_integrity_generate(bio);
  364. return 0;
  365. }
  366. EXPORT_SYMBOL(bio_integrity_prep);
  367. /**
  368. * bio_integrity_verify - Verify integrity metadata for a bio
  369. * @bio: bio to verify
  370. *
  371. * Description: This function is called to verify the integrity of a
  372. * bio. The data in the bio io_vec is compared to the integrity
  373. * metadata returned by the HBA.
  374. */
  375. static int bio_integrity_verify(struct bio *bio)
  376. {
  377. struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
  378. struct blk_integrity_exchg bix;
  379. struct bio_vec *bv;
  380. sector_t sector = bio->bi_integrity->bip_sector;
  381. unsigned int i, sectors, total, ret;
  382. void *prot_buf = bio->bi_integrity->bip_buf;
  383. ret = total = 0;
  384. bix.disk_name = bio->bi_bdev->bd_disk->disk_name;
  385. bix.sector_size = bi->sector_size;
  386. bio_for_each_segment(bv, bio, i) {
  387. void *kaddr = kmap_atomic(bv->bv_page, KM_USER0);
  388. bix.data_buf = kaddr + bv->bv_offset;
  389. bix.data_size = bv->bv_len;
  390. bix.prot_buf = prot_buf;
  391. bix.sector = sector;
  392. ret = bi->verify_fn(&bix);
  393. if (ret) {
  394. kunmap_atomic(kaddr, KM_USER0);
  395. return ret;
  396. }
  397. sectors = bv->bv_len / bi->sector_size;
  398. sector += sectors;
  399. prot_buf += sectors * bi->tuple_size;
  400. total += sectors * bi->tuple_size;
  401. BUG_ON(total > bio->bi_integrity->bip_size);
  402. kunmap_atomic(kaddr, KM_USER0);
  403. }
  404. return ret;
  405. }
  406. /**
  407. * bio_integrity_verify_fn - Integrity I/O completion worker
  408. * @work: Work struct stored in bio to be verified
  409. *
  410. * Description: This workqueue function is called to complete a READ
  411. * request. The function verifies the transferred integrity metadata
  412. * and then calls the original bio end_io function.
  413. */
  414. static void bio_integrity_verify_fn(struct work_struct *work)
  415. {
  416. struct bio_integrity_payload *bip =
  417. container_of(work, struct bio_integrity_payload, bip_work);
  418. struct bio *bio = bip->bip_bio;
  419. int error;
  420. error = bio_integrity_verify(bio);
  421. /* Restore original bio completion handler */
  422. bio->bi_end_io = bip->bip_end_io;
  423. bio_endio(bio, error);
  424. }
  425. /**
  426. * bio_integrity_endio - Integrity I/O completion function
  427. * @bio: Protected bio
  428. * @error: Pointer to errno
  429. *
  430. * Description: Completion for integrity I/O
  431. *
  432. * Normally I/O completion is done in interrupt context. However,
  433. * verifying I/O integrity is a time-consuming task which must be run
  434. * in process context. This function postpones completion
  435. * accordingly.
  436. */
  437. void bio_integrity_endio(struct bio *bio, int error)
  438. {
  439. struct bio_integrity_payload *bip = bio->bi_integrity;
  440. BUG_ON(bip->bip_bio != bio);
  441. /* In case of an I/O error there is no point in verifying the
  442. * integrity metadata. Restore original bio end_io handler
  443. * and run it.
  444. */
  445. if (error) {
  446. bio->bi_end_io = bip->bip_end_io;
  447. bio_endio(bio, error);
  448. return;
  449. }
  450. INIT_WORK(&bip->bip_work, bio_integrity_verify_fn);
  451. queue_work(kintegrityd_wq, &bip->bip_work);
  452. }
  453. EXPORT_SYMBOL(bio_integrity_endio);
  454. /**
  455. * bio_integrity_mark_head - Advance bip_vec skip bytes
  456. * @bip: Integrity vector to advance
  457. * @skip: Number of bytes to advance it
  458. */
  459. void bio_integrity_mark_head(struct bio_integrity_payload *bip,
  460. unsigned int skip)
  461. {
  462. struct bio_vec *iv;
  463. unsigned int i;
  464. bip_for_each_vec(iv, bip, i) {
  465. if (skip == 0) {
  466. bip->bip_idx = i;
  467. return;
  468. } else if (skip >= iv->bv_len) {
  469. skip -= iv->bv_len;
  470. } else { /* skip < iv->bv_len) */
  471. iv->bv_offset += skip;
  472. iv->bv_len -= skip;
  473. bip->bip_idx = i;
  474. return;
  475. }
  476. }
  477. }
  478. /**
  479. * bio_integrity_mark_tail - Truncate bip_vec to be len bytes long
  480. * @bip: Integrity vector to truncate
  481. * @len: New length of integrity vector
  482. */
  483. void bio_integrity_mark_tail(struct bio_integrity_payload *bip,
  484. unsigned int len)
  485. {
  486. struct bio_vec *iv;
  487. unsigned int i;
  488. bip_for_each_vec(iv, bip, i) {
  489. if (len == 0) {
  490. bip->bip_vcnt = i;
  491. return;
  492. } else if (len >= iv->bv_len) {
  493. len -= iv->bv_len;
  494. } else { /* len < iv->bv_len) */
  495. iv->bv_len = len;
  496. len = 0;
  497. }
  498. }
  499. }
  500. /**
  501. * bio_integrity_advance - Advance integrity vector
  502. * @bio: bio whose integrity vector to update
  503. * @bytes_done: number of data bytes that have been completed
  504. *
  505. * Description: This function calculates how many integrity bytes the
  506. * number of completed data bytes correspond to and advances the
  507. * integrity vector accordingly.
  508. */
  509. void bio_integrity_advance(struct bio *bio, unsigned int bytes_done)
  510. {
  511. struct bio_integrity_payload *bip = bio->bi_integrity;
  512. struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
  513. unsigned int nr_sectors;
  514. BUG_ON(bip == NULL);
  515. BUG_ON(bi == NULL);
  516. nr_sectors = bio_integrity_hw_sectors(bi, bytes_done >> 9);
  517. bio_integrity_mark_head(bip, nr_sectors * bi->tuple_size);
  518. }
  519. EXPORT_SYMBOL(bio_integrity_advance);
  520. /**
  521. * bio_integrity_trim - Trim integrity vector
  522. * @bio: bio whose integrity vector to update
  523. * @offset: offset to first data sector
  524. * @sectors: number of data sectors
  525. *
  526. * Description: Used to trim the integrity vector in a cloned bio.
  527. * The ivec will be advanced corresponding to 'offset' data sectors
  528. * and the length will be truncated corresponding to 'len' data
  529. * sectors.
  530. */
  531. void bio_integrity_trim(struct bio *bio, unsigned int offset,
  532. unsigned int sectors)
  533. {
  534. struct bio_integrity_payload *bip = bio->bi_integrity;
  535. struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
  536. unsigned int nr_sectors;
  537. BUG_ON(bip == NULL);
  538. BUG_ON(bi == NULL);
  539. BUG_ON(!bio_flagged(bio, BIO_CLONED));
  540. nr_sectors = bio_integrity_hw_sectors(bi, sectors);
  541. bip->bip_sector = bip->bip_sector + offset;
  542. bio_integrity_mark_head(bip, offset * bi->tuple_size);
  543. bio_integrity_mark_tail(bip, sectors * bi->tuple_size);
  544. }
  545. EXPORT_SYMBOL(bio_integrity_trim);
  546. /**
  547. * bio_integrity_split - Split integrity metadata
  548. * @bio: Protected bio
  549. * @bp: Resulting bio_pair
  550. * @sectors: Offset
  551. *
  552. * Description: Splits an integrity page into a bio_pair.
  553. */
  554. void bio_integrity_split(struct bio *bio, struct bio_pair *bp, int sectors)
  555. {
  556. struct blk_integrity *bi;
  557. struct bio_integrity_payload *bip = bio->bi_integrity;
  558. unsigned int nr_sectors;
  559. if (bio_integrity(bio) == 0)
  560. return;
  561. bi = bdev_get_integrity(bio->bi_bdev);
  562. BUG_ON(bi == NULL);
  563. BUG_ON(bip->bip_vcnt != 1);
  564. nr_sectors = bio_integrity_hw_sectors(bi, sectors);
  565. bp->bio1.bi_integrity = &bp->bip1;
  566. bp->bio2.bi_integrity = &bp->bip2;
  567. bp->iv1 = bip->bip_vec[0];
  568. bp->iv2 = bip->bip_vec[0];
  569. bp->bip1.bip_vec = &bp->iv1;
  570. bp->bip2.bip_vec = &bp->iv2;
  571. bp->iv1.bv_len = sectors * bi->tuple_size;
  572. bp->iv2.bv_offset += sectors * bi->tuple_size;
  573. bp->iv2.bv_len -= sectors * bi->tuple_size;
  574. bp->bip1.bip_sector = bio->bi_integrity->bip_sector;
  575. bp->bip2.bip_sector = bio->bi_integrity->bip_sector + nr_sectors;
  576. bp->bip1.bip_vcnt = bp->bip2.bip_vcnt = 1;
  577. bp->bip1.bip_idx = bp->bip2.bip_idx = 0;
  578. }
  579. EXPORT_SYMBOL(bio_integrity_split);
  580. /**
  581. * bio_integrity_clone - Callback for cloning bios with integrity metadata
  582. * @bio: New bio
  583. * @bio_src: Original bio
  584. * @bs: bio_set to allocate bip from
  585. *
  586. * Description: Called to allocate a bip when cloning a bio
  587. */
  588. int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
  589. struct bio_set *bs)
  590. {
  591. struct bio_integrity_payload *bip_src = bio_src->bi_integrity;
  592. struct bio_integrity_payload *bip;
  593. BUG_ON(bip_src == NULL);
  594. bip = bio_integrity_alloc_bioset(bio, GFP_NOIO, bip_src->bip_vcnt, bs);
  595. if (bip == NULL)
  596. return -EIO;
  597. memcpy(bip->bip_vec, bip_src->bip_vec,
  598. bip_src->bip_vcnt * sizeof(struct bio_vec));
  599. bip->bip_sector = bip_src->bip_sector;
  600. bip->bip_vcnt = bip_src->bip_vcnt;
  601. bip->bip_idx = bip_src->bip_idx;
  602. return 0;
  603. }
  604. EXPORT_SYMBOL(bio_integrity_clone);
  605. int bioset_integrity_create(struct bio_set *bs, int pool_size)
  606. {
  607. bs->bio_integrity_pool = mempool_create_slab_pool(pool_size,
  608. bio_integrity_slab);
  609. if (!bs->bio_integrity_pool)
  610. return -1;
  611. return 0;
  612. }
  613. EXPORT_SYMBOL(bioset_integrity_create);
  614. void bioset_integrity_free(struct bio_set *bs)
  615. {
  616. if (bs->bio_integrity_pool)
  617. mempool_destroy(bs->bio_integrity_pool);
  618. }
  619. EXPORT_SYMBOL(bioset_integrity_free);
  620. void __init bio_integrity_init_slab(void)
  621. {
  622. bio_integrity_slab = KMEM_CACHE(bio_integrity_payload,
  623. SLAB_HWCACHE_ALIGN|SLAB_PANIC);
  624. }
  625. static int __init integrity_init(void)
  626. {
  627. kintegrityd_wq = create_workqueue("kintegrityd");
  628. if (!kintegrityd_wq)
  629. panic("Failed to create kintegrityd\n");
  630. return 0;
  631. }
  632. subsys_initcall(integrity_init);