bio.c 48 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029
  1. /*
  2. * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk>
  3. *
  4. * This program is free software; you can redistribute it and/or modify
  5. * it under the terms of the GNU General Public License version 2 as
  6. * published by the Free Software Foundation.
  7. *
  8. * This program is distributed in the hope that it will be useful,
  9. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11. * GNU General Public License for more details.
  12. *
  13. * You should have received a copy of the GNU General Public Licens
  14. * along with this program; if not, write to the Free Software
  15. * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-
  16. *
  17. */
  18. #include <linux/mm.h>
  19. #include <linux/swap.h>
  20. #include <linux/bio.h>
  21. #include <linux/blkdev.h>
  22. #include <linux/uio.h>
  23. #include <linux/iocontext.h>
  24. #include <linux/slab.h>
  25. #include <linux/init.h>
  26. #include <linux/kernel.h>
  27. #include <linux/export.h>
  28. #include <linux/mempool.h>
  29. #include <linux/workqueue.h>
  30. #include <linux/cgroup.h>
  31. #include <scsi/sg.h> /* for struct sg_iovec */
  32. #include <trace/events/block.h>
  33. /*
  34. * Test patch to inline a certain number of bi_io_vec's inside the bio
  35. * itself, to shrink a bio data allocation from two mempool calls to one
  36. */
  37. #define BIO_INLINE_VECS 4
  38. static mempool_t *bio_split_pool __read_mostly;
  39. /*
  40. * if you change this list, also change bvec_alloc or things will
  41. * break badly! cannot be bigger than what you can fit into an
  42. * unsigned short
  43. */
  44. #define BV(x) { .nr_vecs = x, .name = "biovec-"__stringify(x) }
  45. static struct biovec_slab bvec_slabs[BIOVEC_NR_POOLS] __read_mostly = {
  46. BV(1), BV(4), BV(16), BV(64), BV(128), BV(BIO_MAX_PAGES),
  47. };
  48. #undef BV
  49. /*
  50. * fs_bio_set is the bio_set containing bio and iovec memory pools used by
  51. * IO code that does not need private memory pools.
  52. */
  53. struct bio_set *fs_bio_set;
  54. EXPORT_SYMBOL(fs_bio_set);
  55. /*
  56. * Our slab pool management
  57. */
  58. struct bio_slab {
  59. struct kmem_cache *slab;
  60. unsigned int slab_ref;
  61. unsigned int slab_size;
  62. char name[8];
  63. };
  64. static DEFINE_MUTEX(bio_slab_lock);
  65. static struct bio_slab *bio_slabs;
  66. static unsigned int bio_slab_nr, bio_slab_max;
  67. static struct kmem_cache *bio_find_or_create_slab(unsigned int extra_size)
  68. {
  69. unsigned int sz = sizeof(struct bio) + extra_size;
  70. struct kmem_cache *slab = NULL;
  71. struct bio_slab *bslab, *new_bio_slabs;
  72. unsigned int new_bio_slab_max;
  73. unsigned int i, entry = -1;
  74. mutex_lock(&bio_slab_lock);
  75. i = 0;
  76. while (i < bio_slab_nr) {
  77. bslab = &bio_slabs[i];
  78. if (!bslab->slab && entry == -1)
  79. entry = i;
  80. else if (bslab->slab_size == sz) {
  81. slab = bslab->slab;
  82. bslab->slab_ref++;
  83. break;
  84. }
  85. i++;
  86. }
  87. if (slab)
  88. goto out_unlock;
  89. if (bio_slab_nr == bio_slab_max && entry == -1) {
  90. new_bio_slab_max = bio_slab_max << 1;
  91. new_bio_slabs = krealloc(bio_slabs,
  92. new_bio_slab_max * sizeof(struct bio_slab),
  93. GFP_KERNEL);
  94. if (!new_bio_slabs)
  95. goto out_unlock;
  96. bio_slab_max = new_bio_slab_max;
  97. bio_slabs = new_bio_slabs;
  98. }
  99. if (entry == -1)
  100. entry = bio_slab_nr++;
  101. bslab = &bio_slabs[entry];
  102. snprintf(bslab->name, sizeof(bslab->name), "bio-%d", entry);
  103. slab = kmem_cache_create(bslab->name, sz, 0, SLAB_HWCACHE_ALIGN, NULL);
  104. if (!slab)
  105. goto out_unlock;
  106. printk(KERN_INFO "bio: create slab <%s> at %d\n", bslab->name, entry);
  107. bslab->slab = slab;
  108. bslab->slab_ref = 1;
  109. bslab->slab_size = sz;
  110. out_unlock:
  111. mutex_unlock(&bio_slab_lock);
  112. return slab;
  113. }
  114. static void bio_put_slab(struct bio_set *bs)
  115. {
  116. struct bio_slab *bslab = NULL;
  117. unsigned int i;
  118. mutex_lock(&bio_slab_lock);
  119. for (i = 0; i < bio_slab_nr; i++) {
  120. if (bs->bio_slab == bio_slabs[i].slab) {
  121. bslab = &bio_slabs[i];
  122. break;
  123. }
  124. }
  125. if (WARN(!bslab, KERN_ERR "bio: unable to find slab!\n"))
  126. goto out;
  127. WARN_ON(!bslab->slab_ref);
  128. if (--bslab->slab_ref)
  129. goto out;
  130. kmem_cache_destroy(bslab->slab);
  131. bslab->slab = NULL;
  132. out:
  133. mutex_unlock(&bio_slab_lock);
  134. }
  135. unsigned int bvec_nr_vecs(unsigned short idx)
  136. {
  137. return bvec_slabs[idx].nr_vecs;
  138. }
  139. void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned int idx)
  140. {
  141. BIO_BUG_ON(idx >= BIOVEC_NR_POOLS);
  142. if (idx == BIOVEC_MAX_IDX)
  143. mempool_free(bv, pool);
  144. else {
  145. struct biovec_slab *bvs = bvec_slabs + idx;
  146. kmem_cache_free(bvs->slab, bv);
  147. }
  148. }
  149. struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx,
  150. mempool_t *pool)
  151. {
  152. struct bio_vec *bvl;
  153. /*
  154. * see comment near bvec_array define!
  155. */
  156. switch (nr) {
  157. case 1:
  158. *idx = 0;
  159. break;
  160. case 2 ... 4:
  161. *idx = 1;
  162. break;
  163. case 5 ... 16:
  164. *idx = 2;
  165. break;
  166. case 17 ... 64:
  167. *idx = 3;
  168. break;
  169. case 65 ... 128:
  170. *idx = 4;
  171. break;
  172. case 129 ... BIO_MAX_PAGES:
  173. *idx = 5;
  174. break;
  175. default:
  176. return NULL;
  177. }
  178. /*
  179. * idx now points to the pool we want to allocate from. only the
  180. * 1-vec entry pool is mempool backed.
  181. */
  182. if (*idx == BIOVEC_MAX_IDX) {
  183. fallback:
  184. bvl = mempool_alloc(pool, gfp_mask);
  185. } else {
  186. struct biovec_slab *bvs = bvec_slabs + *idx;
  187. gfp_t __gfp_mask = gfp_mask & ~(__GFP_WAIT | __GFP_IO);
  188. /*
  189. * Make this allocation restricted and don't dump info on
  190. * allocation failures, since we'll fallback to the mempool
  191. * in case of failure.
  192. */
  193. __gfp_mask |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
  194. /*
  195. * Try a slab allocation. If this fails and __GFP_WAIT
  196. * is set, retry with the 1-entry mempool
  197. */
  198. bvl = kmem_cache_alloc(bvs->slab, __gfp_mask);
  199. if (unlikely(!bvl && (gfp_mask & __GFP_WAIT))) {
  200. *idx = BIOVEC_MAX_IDX;
  201. goto fallback;
  202. }
  203. }
  204. return bvl;
  205. }
  206. static void __bio_free(struct bio *bio)
  207. {
  208. bio_disassociate_task(bio);
  209. if (bio_integrity(bio))
  210. bio_integrity_free(bio);
  211. }
  212. static void bio_free(struct bio *bio)
  213. {
  214. struct bio_set *bs = bio->bi_pool;
  215. void *p;
  216. __bio_free(bio);
  217. if (bs) {
  218. if (bio_flagged(bio, BIO_OWNS_VEC))
  219. bvec_free(bs->bvec_pool, bio->bi_io_vec, BIO_POOL_IDX(bio));
  220. /*
  221. * If we have front padding, adjust the bio pointer before freeing
  222. */
  223. p = bio;
  224. p -= bs->front_pad;
  225. mempool_free(p, bs->bio_pool);
  226. } else {
  227. /* Bio was allocated by bio_kmalloc() */
  228. kfree(bio);
  229. }
  230. }
  231. void bio_init(struct bio *bio)
  232. {
  233. memset(bio, 0, sizeof(*bio));
  234. bio->bi_flags = 1 << BIO_UPTODATE;
  235. atomic_set(&bio->bi_cnt, 1);
  236. }
  237. EXPORT_SYMBOL(bio_init);
  238. /**
  239. * bio_reset - reinitialize a bio
  240. * @bio: bio to reset
  241. *
  242. * Description:
  243. * After calling bio_reset(), @bio will be in the same state as a freshly
  244. * allocated bio returned bio bio_alloc_bioset() - the only fields that are
  245. * preserved are the ones that are initialized by bio_alloc_bioset(). See
  246. * comment in struct bio.
  247. */
  248. void bio_reset(struct bio *bio)
  249. {
  250. unsigned long flags = bio->bi_flags & (~0UL << BIO_RESET_BITS);
  251. __bio_free(bio);
  252. memset(bio, 0, BIO_RESET_BYTES);
  253. bio->bi_flags = flags|(1 << BIO_UPTODATE);
  254. }
  255. EXPORT_SYMBOL(bio_reset);
  256. static void bio_alloc_rescue(struct work_struct *work)
  257. {
  258. struct bio_set *bs = container_of(work, struct bio_set, rescue_work);
  259. struct bio *bio;
  260. while (1) {
  261. spin_lock(&bs->rescue_lock);
  262. bio = bio_list_pop(&bs->rescue_list);
  263. spin_unlock(&bs->rescue_lock);
  264. if (!bio)
  265. break;
  266. generic_make_request(bio);
  267. }
  268. }
  269. static void punt_bios_to_rescuer(struct bio_set *bs)
  270. {
  271. struct bio_list punt, nopunt;
  272. struct bio *bio;
  273. /*
  274. * In order to guarantee forward progress we must punt only bios that
  275. * were allocated from this bio_set; otherwise, if there was a bio on
  276. * there for a stacking driver higher up in the stack, processing it
  277. * could require allocating bios from this bio_set, and doing that from
  278. * our own rescuer would be bad.
  279. *
  280. * Since bio lists are singly linked, pop them all instead of trying to
  281. * remove from the middle of the list:
  282. */
  283. bio_list_init(&punt);
  284. bio_list_init(&nopunt);
  285. while ((bio = bio_list_pop(current->bio_list)))
  286. bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
  287. *current->bio_list = nopunt;
  288. spin_lock(&bs->rescue_lock);
  289. bio_list_merge(&bs->rescue_list, &punt);
  290. spin_unlock(&bs->rescue_lock);
  291. queue_work(bs->rescue_workqueue, &bs->rescue_work);
  292. }
  293. /**
  294. * bio_alloc_bioset - allocate a bio for I/O
  295. * @gfp_mask: the GFP_ mask given to the slab allocator
  296. * @nr_iovecs: number of iovecs to pre-allocate
  297. * @bs: the bio_set to allocate from.
  298. *
  299. * Description:
  300. * If @bs is NULL, uses kmalloc() to allocate the bio; else the allocation is
  301. * backed by the @bs's mempool.
  302. *
  303. * When @bs is not NULL, if %__GFP_WAIT is set then bio_alloc will always be
  304. * able to allocate a bio. This is due to the mempool guarantees. To make this
  305. * work, callers must never allocate more than 1 bio at a time from this pool.
  306. * Callers that need to allocate more than 1 bio must always submit the
  307. * previously allocated bio for IO before attempting to allocate a new one.
  308. * Failure to do so can cause deadlocks under memory pressure.
  309. *
  310. * Note that when running under generic_make_request() (i.e. any block
  311. * driver), bios are not submitted until after you return - see the code in
  312. * generic_make_request() that converts recursion into iteration, to prevent
  313. * stack overflows.
  314. *
  315. * This would normally mean allocating multiple bios under
  316. * generic_make_request() would be susceptible to deadlocks, but we have
  317. * deadlock avoidance code that resubmits any blocked bios from a rescuer
  318. * thread.
  319. *
  320. * However, we do not guarantee forward progress for allocations from other
  321. * mempools. Doing multiple allocations from the same mempool under
  322. * generic_make_request() should be avoided - instead, use bio_set's front_pad
  323. * for per bio allocations.
  324. *
  325. * RETURNS:
  326. * Pointer to new bio on success, NULL on failure.
  327. */
  328. struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs)
  329. {
  330. gfp_t saved_gfp = gfp_mask;
  331. unsigned front_pad;
  332. unsigned inline_vecs;
  333. unsigned long idx = BIO_POOL_NONE;
  334. struct bio_vec *bvl = NULL;
  335. struct bio *bio;
  336. void *p;
  337. if (!bs) {
  338. if (nr_iovecs > UIO_MAXIOV)
  339. return NULL;
  340. p = kmalloc(sizeof(struct bio) +
  341. nr_iovecs * sizeof(struct bio_vec),
  342. gfp_mask);
  343. front_pad = 0;
  344. inline_vecs = nr_iovecs;
  345. } else {
  346. /*
  347. * generic_make_request() converts recursion to iteration; this
  348. * means if we're running beneath it, any bios we allocate and
  349. * submit will not be submitted (and thus freed) until after we
  350. * return.
  351. *
  352. * This exposes us to a potential deadlock if we allocate
  353. * multiple bios from the same bio_set() while running
  354. * underneath generic_make_request(). If we were to allocate
  355. * multiple bios (say a stacking block driver that was splitting
  356. * bios), we would deadlock if we exhausted the mempool's
  357. * reserve.
  358. *
  359. * We solve this, and guarantee forward progress, with a rescuer
  360. * workqueue per bio_set. If we go to allocate and there are
  361. * bios on current->bio_list, we first try the allocation
  362. * without __GFP_WAIT; if that fails, we punt those bios we
  363. * would be blocking to the rescuer workqueue before we retry
  364. * with the original gfp_flags.
  365. */
  366. if (current->bio_list && !bio_list_empty(current->bio_list))
  367. gfp_mask &= ~__GFP_WAIT;
  368. p = mempool_alloc(bs->bio_pool, gfp_mask);
  369. if (!p && gfp_mask != saved_gfp) {
  370. punt_bios_to_rescuer(bs);
  371. gfp_mask = saved_gfp;
  372. p = mempool_alloc(bs->bio_pool, gfp_mask);
  373. }
  374. front_pad = bs->front_pad;
  375. inline_vecs = BIO_INLINE_VECS;
  376. }
  377. if (unlikely(!p))
  378. return NULL;
  379. bio = p + front_pad;
  380. bio_init(bio);
  381. if (nr_iovecs > inline_vecs) {
  382. bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);
  383. if (!bvl && gfp_mask != saved_gfp) {
  384. punt_bios_to_rescuer(bs);
  385. gfp_mask = saved_gfp;
  386. bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);
  387. }
  388. if (unlikely(!bvl))
  389. goto err_free;
  390. bio->bi_flags |= 1 << BIO_OWNS_VEC;
  391. } else if (nr_iovecs) {
  392. bvl = bio->bi_inline_vecs;
  393. }
  394. bio->bi_pool = bs;
  395. bio->bi_flags |= idx << BIO_POOL_OFFSET;
  396. bio->bi_max_vecs = nr_iovecs;
  397. bio->bi_io_vec = bvl;
  398. return bio;
  399. err_free:
  400. mempool_free(p, bs->bio_pool);
  401. return NULL;
  402. }
  403. EXPORT_SYMBOL(bio_alloc_bioset);
  404. void zero_fill_bio(struct bio *bio)
  405. {
  406. unsigned long flags;
  407. struct bio_vec *bv;
  408. int i;
  409. bio_for_each_segment(bv, bio, i) {
  410. char *data = bvec_kmap_irq(bv, &flags);
  411. memset(data, 0, bv->bv_len);
  412. flush_dcache_page(bv->bv_page);
  413. bvec_kunmap_irq(data, &flags);
  414. }
  415. }
  416. EXPORT_SYMBOL(zero_fill_bio);
  417. /**
  418. * bio_put - release a reference to a bio
  419. * @bio: bio to release reference to
  420. *
  421. * Description:
  422. * Put a reference to a &struct bio, either one you have gotten with
  423. * bio_alloc, bio_get or bio_clone. The last put of a bio will free it.
  424. **/
  425. void bio_put(struct bio *bio)
  426. {
  427. BIO_BUG_ON(!atomic_read(&bio->bi_cnt));
  428. /*
  429. * last put frees it
  430. */
  431. if (atomic_dec_and_test(&bio->bi_cnt))
  432. bio_free(bio);
  433. }
  434. EXPORT_SYMBOL(bio_put);
  435. inline int bio_phys_segments(struct request_queue *q, struct bio *bio)
  436. {
  437. if (unlikely(!bio_flagged(bio, BIO_SEG_VALID)))
  438. blk_recount_segments(q, bio);
  439. return bio->bi_phys_segments;
  440. }
  441. EXPORT_SYMBOL(bio_phys_segments);
  442. /**
  443. * __bio_clone - clone a bio
  444. * @bio: destination bio
  445. * @bio_src: bio to clone
  446. *
  447. * Clone a &bio. Caller will own the returned bio, but not
  448. * the actual data it points to. Reference count of returned
  449. * bio will be one.
  450. */
  451. void __bio_clone(struct bio *bio, struct bio *bio_src)
  452. {
  453. memcpy(bio->bi_io_vec, bio_src->bi_io_vec,
  454. bio_src->bi_max_vecs * sizeof(struct bio_vec));
  455. /*
  456. * most users will be overriding ->bi_bdev with a new target,
  457. * so we don't set nor calculate new physical/hw segment counts here
  458. */
  459. bio->bi_sector = bio_src->bi_sector;
  460. bio->bi_bdev = bio_src->bi_bdev;
  461. bio->bi_flags |= 1 << BIO_CLONED;
  462. bio->bi_rw = bio_src->bi_rw;
  463. bio->bi_vcnt = bio_src->bi_vcnt;
  464. bio->bi_size = bio_src->bi_size;
  465. bio->bi_idx = bio_src->bi_idx;
  466. }
  467. EXPORT_SYMBOL(__bio_clone);
  468. /**
  469. * bio_clone_bioset - clone a bio
  470. * @bio: bio to clone
  471. * @gfp_mask: allocation priority
  472. * @bs: bio_set to allocate from
  473. *
  474. * Like __bio_clone, only also allocates the returned bio
  475. */
  476. struct bio *bio_clone_bioset(struct bio *bio, gfp_t gfp_mask,
  477. struct bio_set *bs)
  478. {
  479. struct bio *b;
  480. b = bio_alloc_bioset(gfp_mask, bio->bi_max_vecs, bs);
  481. if (!b)
  482. return NULL;
  483. __bio_clone(b, bio);
  484. if (bio_integrity(bio)) {
  485. int ret;
  486. ret = bio_integrity_clone(b, bio, gfp_mask);
  487. if (ret < 0) {
  488. bio_put(b);
  489. return NULL;
  490. }
  491. }
  492. return b;
  493. }
  494. EXPORT_SYMBOL(bio_clone_bioset);
  495. /**
  496. * bio_get_nr_vecs - return approx number of vecs
  497. * @bdev: I/O target
  498. *
  499. * Return the approximate number of pages we can send to this target.
  500. * There's no guarantee that you will be able to fit this number of pages
  501. * into a bio, it does not account for dynamic restrictions that vary
  502. * on offset.
  503. */
  504. int bio_get_nr_vecs(struct block_device *bdev)
  505. {
  506. struct request_queue *q = bdev_get_queue(bdev);
  507. int nr_pages;
  508. nr_pages = min_t(unsigned,
  509. queue_max_segments(q),
  510. queue_max_sectors(q) / (PAGE_SIZE >> 9) + 1);
  511. return min_t(unsigned, nr_pages, BIO_MAX_PAGES);
  512. }
  513. EXPORT_SYMBOL(bio_get_nr_vecs);
  514. static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
  515. *page, unsigned int len, unsigned int offset,
  516. unsigned short max_sectors)
  517. {
  518. int retried_segments = 0;
  519. struct bio_vec *bvec;
  520. /*
  521. * cloned bio must not modify vec list
  522. */
  523. if (unlikely(bio_flagged(bio, BIO_CLONED)))
  524. return 0;
  525. if (((bio->bi_size + len) >> 9) > max_sectors)
  526. return 0;
  527. /*
  528. * For filesystems with a blocksize smaller than the pagesize
  529. * we will often be called with the same page as last time and
  530. * a consecutive offset. Optimize this special case.
  531. */
  532. if (bio->bi_vcnt > 0) {
  533. struct bio_vec *prev = &bio->bi_io_vec[bio->bi_vcnt - 1];
  534. if (page == prev->bv_page &&
  535. offset == prev->bv_offset + prev->bv_len) {
  536. unsigned int prev_bv_len = prev->bv_len;
  537. prev->bv_len += len;
  538. if (q->merge_bvec_fn) {
  539. struct bvec_merge_data bvm = {
  540. /* prev_bvec is already charged in
  541. bi_size, discharge it in order to
  542. simulate merging updated prev_bvec
  543. as new bvec. */
  544. .bi_bdev = bio->bi_bdev,
  545. .bi_sector = bio->bi_sector,
  546. .bi_size = bio->bi_size - prev_bv_len,
  547. .bi_rw = bio->bi_rw,
  548. };
  549. if (q->merge_bvec_fn(q, &bvm, prev) < prev->bv_len) {
  550. prev->bv_len -= len;
  551. return 0;
  552. }
  553. }
  554. goto done;
  555. }
  556. }
  557. if (bio->bi_vcnt >= bio->bi_max_vecs)
  558. return 0;
  559. /*
  560. * we might lose a segment or two here, but rather that than
  561. * make this too complex.
  562. */
  563. while (bio->bi_phys_segments >= queue_max_segments(q)) {
  564. if (retried_segments)
  565. return 0;
  566. retried_segments = 1;
  567. blk_recount_segments(q, bio);
  568. }
  569. /*
  570. * setup the new entry, we might clear it again later if we
  571. * cannot add the page
  572. */
  573. bvec = &bio->bi_io_vec[bio->bi_vcnt];
  574. bvec->bv_page = page;
  575. bvec->bv_len = len;
  576. bvec->bv_offset = offset;
  577. /*
  578. * if queue has other restrictions (eg varying max sector size
  579. * depending on offset), it can specify a merge_bvec_fn in the
  580. * queue to get further control
  581. */
  582. if (q->merge_bvec_fn) {
  583. struct bvec_merge_data bvm = {
  584. .bi_bdev = bio->bi_bdev,
  585. .bi_sector = bio->bi_sector,
  586. .bi_size = bio->bi_size,
  587. .bi_rw = bio->bi_rw,
  588. };
  589. /*
  590. * merge_bvec_fn() returns number of bytes it can accept
  591. * at this offset
  592. */
  593. if (q->merge_bvec_fn(q, &bvm, bvec) < bvec->bv_len) {
  594. bvec->bv_page = NULL;
  595. bvec->bv_len = 0;
  596. bvec->bv_offset = 0;
  597. return 0;
  598. }
  599. }
  600. /* If we may be able to merge these biovecs, force a recount */
  601. if (bio->bi_vcnt && (BIOVEC_PHYS_MERGEABLE(bvec-1, bvec)))
  602. bio->bi_flags &= ~(1 << BIO_SEG_VALID);
  603. bio->bi_vcnt++;
  604. bio->bi_phys_segments++;
  605. done:
  606. bio->bi_size += len;
  607. return len;
  608. }
  609. /**
  610. * bio_add_pc_page - attempt to add page to bio
  611. * @q: the target queue
  612. * @bio: destination bio
  613. * @page: page to add
  614. * @len: vec entry length
  615. * @offset: vec entry offset
  616. *
  617. * Attempt to add a page to the bio_vec maplist. This can fail for a
  618. * number of reasons, such as the bio being full or target block device
  619. * limitations. The target block device must allow bio's up to PAGE_SIZE,
  620. * so it is always possible to add a single page to an empty bio.
  621. *
  622. * This should only be used by REQ_PC bios.
  623. */
  624. int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page *page,
  625. unsigned int len, unsigned int offset)
  626. {
  627. return __bio_add_page(q, bio, page, len, offset,
  628. queue_max_hw_sectors(q));
  629. }
  630. EXPORT_SYMBOL(bio_add_pc_page);
  631. /**
  632. * bio_add_page - attempt to add page to bio
  633. * @bio: destination bio
  634. * @page: page to add
  635. * @len: vec entry length
  636. * @offset: vec entry offset
  637. *
  638. * Attempt to add a page to the bio_vec maplist. This can fail for a
  639. * number of reasons, such as the bio being full or target block device
  640. * limitations. The target block device must allow bio's up to PAGE_SIZE,
  641. * so it is always possible to add a single page to an empty bio.
  642. */
  643. int bio_add_page(struct bio *bio, struct page *page, unsigned int len,
  644. unsigned int offset)
  645. {
  646. struct request_queue *q = bdev_get_queue(bio->bi_bdev);
  647. return __bio_add_page(q, bio, page, len, offset, queue_max_sectors(q));
  648. }
  649. EXPORT_SYMBOL(bio_add_page);
  650. struct submit_bio_ret {
  651. struct completion event;
  652. int error;
  653. };
  654. static void submit_bio_wait_endio(struct bio *bio, int error)
  655. {
  656. struct submit_bio_ret *ret = bio->bi_private;
  657. ret->error = error;
  658. complete(&ret->event);
  659. }
  660. /**
  661. * submit_bio_wait - submit a bio, and wait until it completes
  662. * @rw: whether to %READ or %WRITE, or maybe to %READA (read ahead)
  663. * @bio: The &struct bio which describes the I/O
  664. *
  665. * Simple wrapper around submit_bio(). Returns 0 on success, or the error from
  666. * bio_endio() on failure.
  667. */
  668. int submit_bio_wait(int rw, struct bio *bio)
  669. {
  670. struct submit_bio_ret ret;
  671. rw |= REQ_SYNC;
  672. init_completion(&ret.event);
  673. bio->bi_private = &ret;
  674. bio->bi_end_io = submit_bio_wait_endio;
  675. submit_bio(rw, bio);
  676. wait_for_completion(&ret.event);
  677. return ret.error;
  678. }
  679. EXPORT_SYMBOL(submit_bio_wait);
  680. /**
  681. * bio_advance - increment/complete a bio by some number of bytes
  682. * @bio: bio to advance
  683. * @bytes: number of bytes to complete
  684. *
  685. * This updates bi_sector, bi_size and bi_idx; if the number of bytes to
  686. * complete doesn't align with a bvec boundary, then bv_len and bv_offset will
  687. * be updated on the last bvec as well.
  688. *
  689. * @bio will then represent the remaining, uncompleted portion of the io.
  690. */
  691. void bio_advance(struct bio *bio, unsigned bytes)
  692. {
  693. if (bio_integrity(bio))
  694. bio_integrity_advance(bio, bytes);
  695. bio->bi_sector += bytes >> 9;
  696. bio->bi_size -= bytes;
  697. if (bio->bi_rw & BIO_NO_ADVANCE_ITER_MASK)
  698. return;
  699. while (bytes) {
  700. if (unlikely(bio->bi_idx >= bio->bi_vcnt)) {
  701. WARN_ONCE(1, "bio idx %d >= vcnt %d\n",
  702. bio->bi_idx, bio->bi_vcnt);
  703. break;
  704. }
  705. if (bytes >= bio_iovec(bio)->bv_len) {
  706. bytes -= bio_iovec(bio)->bv_len;
  707. bio->bi_idx++;
  708. } else {
  709. bio_iovec(bio)->bv_len -= bytes;
  710. bio_iovec(bio)->bv_offset += bytes;
  711. bytes = 0;
  712. }
  713. }
  714. }
  715. EXPORT_SYMBOL(bio_advance);
  716. /**
  717. * bio_alloc_pages - allocates a single page for each bvec in a bio
  718. * @bio: bio to allocate pages for
  719. * @gfp_mask: flags for allocation
  720. *
  721. * Allocates pages up to @bio->bi_vcnt.
  722. *
  723. * Returns 0 on success, -ENOMEM on failure. On failure, any allocated pages are
  724. * freed.
  725. */
  726. int bio_alloc_pages(struct bio *bio, gfp_t gfp_mask)
  727. {
  728. int i;
  729. struct bio_vec *bv;
  730. bio_for_each_segment_all(bv, bio, i) {
  731. bv->bv_page = alloc_page(gfp_mask);
  732. if (!bv->bv_page) {
  733. while (--bv >= bio->bi_io_vec)
  734. __free_page(bv->bv_page);
  735. return -ENOMEM;
  736. }
  737. }
  738. return 0;
  739. }
  740. EXPORT_SYMBOL(bio_alloc_pages);
  741. /**
  742. * bio_copy_data - copy contents of data buffers from one chain of bios to
  743. * another
  744. * @src: source bio list
  745. * @dst: destination bio list
  746. *
  747. * If @src and @dst are single bios, bi_next must be NULL - otherwise, treats
  748. * @src and @dst as linked lists of bios.
  749. *
  750. * Stops when it reaches the end of either @src or @dst - that is, copies
  751. * min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of bios).
  752. */
  753. void bio_copy_data(struct bio *dst, struct bio *src)
  754. {
  755. struct bio_vec *src_bv, *dst_bv;
  756. unsigned src_offset, dst_offset, bytes;
  757. void *src_p, *dst_p;
  758. src_bv = bio_iovec(src);
  759. dst_bv = bio_iovec(dst);
  760. src_offset = src_bv->bv_offset;
  761. dst_offset = dst_bv->bv_offset;
  762. while (1) {
  763. if (src_offset == src_bv->bv_offset + src_bv->bv_len) {
  764. src_bv++;
  765. if (src_bv == bio_iovec_idx(src, src->bi_vcnt)) {
  766. src = src->bi_next;
  767. if (!src)
  768. break;
  769. src_bv = bio_iovec(src);
  770. }
  771. src_offset = src_bv->bv_offset;
  772. }
  773. if (dst_offset == dst_bv->bv_offset + dst_bv->bv_len) {
  774. dst_bv++;
  775. if (dst_bv == bio_iovec_idx(dst, dst->bi_vcnt)) {
  776. dst = dst->bi_next;
  777. if (!dst)
  778. break;
  779. dst_bv = bio_iovec(dst);
  780. }
  781. dst_offset = dst_bv->bv_offset;
  782. }
  783. bytes = min(dst_bv->bv_offset + dst_bv->bv_len - dst_offset,
  784. src_bv->bv_offset + src_bv->bv_len - src_offset);
  785. src_p = kmap_atomic(src_bv->bv_page);
  786. dst_p = kmap_atomic(dst_bv->bv_page);
  787. memcpy(dst_p + dst_offset,
  788. src_p + src_offset,
  789. bytes);
  790. kunmap_atomic(dst_p);
  791. kunmap_atomic(src_p);
  792. src_offset += bytes;
  793. dst_offset += bytes;
  794. }
  795. }
  796. EXPORT_SYMBOL(bio_copy_data);
  797. struct bio_map_data {
  798. struct bio_vec *iovecs;
  799. struct sg_iovec *sgvecs;
  800. int nr_sgvecs;
  801. int is_our_pages;
  802. };
  803. static void bio_set_map_data(struct bio_map_data *bmd, struct bio *bio,
  804. struct sg_iovec *iov, int iov_count,
  805. int is_our_pages)
  806. {
  807. memcpy(bmd->iovecs, bio->bi_io_vec, sizeof(struct bio_vec) * bio->bi_vcnt);
  808. memcpy(bmd->sgvecs, iov, sizeof(struct sg_iovec) * iov_count);
  809. bmd->nr_sgvecs = iov_count;
  810. bmd->is_our_pages = is_our_pages;
  811. bio->bi_private = bmd;
  812. }
  813. static void bio_free_map_data(struct bio_map_data *bmd)
  814. {
  815. kfree(bmd->iovecs);
  816. kfree(bmd->sgvecs);
  817. kfree(bmd);
  818. }
  819. static struct bio_map_data *bio_alloc_map_data(int nr_segs,
  820. unsigned int iov_count,
  821. gfp_t gfp_mask)
  822. {
  823. struct bio_map_data *bmd;
  824. if (iov_count > UIO_MAXIOV)
  825. return NULL;
  826. bmd = kmalloc(sizeof(*bmd), gfp_mask);
  827. if (!bmd)
  828. return NULL;
  829. bmd->iovecs = kmalloc(sizeof(struct bio_vec) * nr_segs, gfp_mask);
  830. if (!bmd->iovecs) {
  831. kfree(bmd);
  832. return NULL;
  833. }
  834. bmd->sgvecs = kmalloc(sizeof(struct sg_iovec) * iov_count, gfp_mask);
  835. if (bmd->sgvecs)
  836. return bmd;
  837. kfree(bmd->iovecs);
  838. kfree(bmd);
  839. return NULL;
  840. }
  841. static int __bio_copy_iov(struct bio *bio, struct bio_vec *iovecs,
  842. struct sg_iovec *iov, int iov_count,
  843. int to_user, int from_user, int do_free_page)
  844. {
  845. int ret = 0, i;
  846. struct bio_vec *bvec;
  847. int iov_idx = 0;
  848. unsigned int iov_off = 0;
  849. bio_for_each_segment_all(bvec, bio, i) {
  850. char *bv_addr = page_address(bvec->bv_page);
  851. unsigned int bv_len = iovecs[i].bv_len;
  852. while (bv_len && iov_idx < iov_count) {
  853. unsigned int bytes;
  854. char __user *iov_addr;
  855. bytes = min_t(unsigned int,
  856. iov[iov_idx].iov_len - iov_off, bv_len);
  857. iov_addr = iov[iov_idx].iov_base + iov_off;
  858. if (!ret) {
  859. if (to_user)
  860. ret = copy_to_user(iov_addr, bv_addr,
  861. bytes);
  862. if (from_user)
  863. ret = copy_from_user(bv_addr, iov_addr,
  864. bytes);
  865. if (ret)
  866. ret = -EFAULT;
  867. }
  868. bv_len -= bytes;
  869. bv_addr += bytes;
  870. iov_addr += bytes;
  871. iov_off += bytes;
  872. if (iov[iov_idx].iov_len == iov_off) {
  873. iov_idx++;
  874. iov_off = 0;
  875. }
  876. }
  877. if (do_free_page)
  878. __free_page(bvec->bv_page);
  879. }
  880. return ret;
  881. }
  882. /**
  883. * bio_uncopy_user - finish previously mapped bio
  884. * @bio: bio being terminated
  885. *
  886. * Free pages allocated from bio_copy_user() and write back data
  887. * to user space in case of a read.
  888. */
  889. int bio_uncopy_user(struct bio *bio)
  890. {
  891. struct bio_map_data *bmd = bio->bi_private;
  892. struct bio_vec *bvec;
  893. int ret = 0, i;
  894. if (!bio_flagged(bio, BIO_NULL_MAPPED)) {
  895. /*
  896. * if we're in a workqueue, the request is orphaned, so
  897. * don't copy into a random user address space, just free.
  898. */
  899. if (current->mm)
  900. ret = __bio_copy_iov(bio, bmd->iovecs, bmd->sgvecs,
  901. bmd->nr_sgvecs, bio_data_dir(bio) == READ,
  902. 0, bmd->is_our_pages);
  903. else if (bmd->is_our_pages)
  904. bio_for_each_segment_all(bvec, bio, i)
  905. __free_page(bvec->bv_page);
  906. }
  907. bio_free_map_data(bmd);
  908. bio_put(bio);
  909. return ret;
  910. }
  911. EXPORT_SYMBOL(bio_uncopy_user);
  912. /**
  913. * bio_copy_user_iov - copy user data to bio
  914. * @q: destination block queue
  915. * @map_data: pointer to the rq_map_data holding pages (if necessary)
  916. * @iov: the iovec.
  917. * @iov_count: number of elements in the iovec
  918. * @write_to_vm: bool indicating writing to pages or not
  919. * @gfp_mask: memory allocation flags
  920. *
  921. * Prepares and returns a bio for indirect user io, bouncing data
  922. * to/from kernel pages as necessary. Must be paired with
  923. * call bio_uncopy_user() on io completion.
  924. */
  925. struct bio *bio_copy_user_iov(struct request_queue *q,
  926. struct rq_map_data *map_data,
  927. struct sg_iovec *iov, int iov_count,
  928. int write_to_vm, gfp_t gfp_mask)
  929. {
  930. struct bio_map_data *bmd;
  931. struct bio_vec *bvec;
  932. struct page *page;
  933. struct bio *bio;
  934. int i, ret;
  935. int nr_pages = 0;
  936. unsigned int len = 0;
  937. unsigned int offset = map_data ? map_data->offset & ~PAGE_MASK : 0;
  938. for (i = 0; i < iov_count; i++) {
  939. unsigned long uaddr;
  940. unsigned long end;
  941. unsigned long start;
  942. uaddr = (unsigned long)iov[i].iov_base;
  943. end = (uaddr + iov[i].iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
  944. start = uaddr >> PAGE_SHIFT;
  945. /*
  946. * Overflow, abort
  947. */
  948. if (end < start)
  949. return ERR_PTR(-EINVAL);
  950. nr_pages += end - start;
  951. len += iov[i].iov_len;
  952. }
  953. if (offset)
  954. nr_pages++;
  955. bmd = bio_alloc_map_data(nr_pages, iov_count, gfp_mask);
  956. if (!bmd)
  957. return ERR_PTR(-ENOMEM);
  958. ret = -ENOMEM;
  959. bio = bio_kmalloc(gfp_mask, nr_pages);
  960. if (!bio)
  961. goto out_bmd;
  962. if (!write_to_vm)
  963. bio->bi_rw |= REQ_WRITE;
  964. ret = 0;
  965. if (map_data) {
  966. nr_pages = 1 << map_data->page_order;
  967. i = map_data->offset / PAGE_SIZE;
  968. }
  969. while (len) {
  970. unsigned int bytes = PAGE_SIZE;
  971. bytes -= offset;
  972. if (bytes > len)
  973. bytes = len;
  974. if (map_data) {
  975. if (i == map_data->nr_entries * nr_pages) {
  976. ret = -ENOMEM;
  977. break;
  978. }
  979. page = map_data->pages[i / nr_pages];
  980. page += (i % nr_pages);
  981. i++;
  982. } else {
  983. page = alloc_page(q->bounce_gfp | gfp_mask);
  984. if (!page) {
  985. ret = -ENOMEM;
  986. break;
  987. }
  988. }
  989. if (bio_add_pc_page(q, bio, page, bytes, offset) < bytes)
  990. break;
  991. len -= bytes;
  992. offset = 0;
  993. }
  994. if (ret)
  995. goto cleanup;
  996. /*
  997. * success
  998. */
  999. if ((!write_to_vm && (!map_data || !map_data->null_mapped)) ||
  1000. (map_data && map_data->from_user)) {
  1001. ret = __bio_copy_iov(bio, bio->bi_io_vec, iov, iov_count, 0, 1, 0);
  1002. if (ret)
  1003. goto cleanup;
  1004. }
  1005. bio_set_map_data(bmd, bio, iov, iov_count, map_data ? 0 : 1);
  1006. return bio;
  1007. cleanup:
  1008. if (!map_data)
  1009. bio_for_each_segment_all(bvec, bio, i)
  1010. __free_page(bvec->bv_page);
  1011. bio_put(bio);
  1012. out_bmd:
  1013. bio_free_map_data(bmd);
  1014. return ERR_PTR(ret);
  1015. }
  1016. /**
  1017. * bio_copy_user - copy user data to bio
  1018. * @q: destination block queue
  1019. * @map_data: pointer to the rq_map_data holding pages (if necessary)
  1020. * @uaddr: start of user address
  1021. * @len: length in bytes
  1022. * @write_to_vm: bool indicating writing to pages or not
  1023. * @gfp_mask: memory allocation flags
  1024. *
  1025. * Prepares and returns a bio for indirect user io, bouncing data
  1026. * to/from kernel pages as necessary. Must be paired with
  1027. * call bio_uncopy_user() on io completion.
  1028. */
  1029. struct bio *bio_copy_user(struct request_queue *q, struct rq_map_data *map_data,
  1030. unsigned long uaddr, unsigned int len,
  1031. int write_to_vm, gfp_t gfp_mask)
  1032. {
  1033. struct sg_iovec iov;
  1034. iov.iov_base = (void __user *)uaddr;
  1035. iov.iov_len = len;
  1036. return bio_copy_user_iov(q, map_data, &iov, 1, write_to_vm, gfp_mask);
  1037. }
  1038. EXPORT_SYMBOL(bio_copy_user);
  1039. static struct bio *__bio_map_user_iov(struct request_queue *q,
  1040. struct block_device *bdev,
  1041. struct sg_iovec *iov, int iov_count,
  1042. int write_to_vm, gfp_t gfp_mask)
  1043. {
  1044. int i, j;
  1045. int nr_pages = 0;
  1046. struct page **pages;
  1047. struct bio *bio;
  1048. int cur_page = 0;
  1049. int ret, offset;
  1050. for (i = 0; i < iov_count; i++) {
  1051. unsigned long uaddr = (unsigned long)iov[i].iov_base;
  1052. unsigned long len = iov[i].iov_len;
  1053. unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
  1054. unsigned long start = uaddr >> PAGE_SHIFT;
  1055. /*
  1056. * Overflow, abort
  1057. */
  1058. if (end < start)
  1059. return ERR_PTR(-EINVAL);
  1060. nr_pages += end - start;
  1061. /*
  1062. * buffer must be aligned to at least hardsector size for now
  1063. */
  1064. if (uaddr & queue_dma_alignment(q))
  1065. return ERR_PTR(-EINVAL);
  1066. }
  1067. if (!nr_pages)
  1068. return ERR_PTR(-EINVAL);
  1069. bio = bio_kmalloc(gfp_mask, nr_pages);
  1070. if (!bio)
  1071. return ERR_PTR(-ENOMEM);
  1072. ret = -ENOMEM;
  1073. pages = kcalloc(nr_pages, sizeof(struct page *), gfp_mask);
  1074. if (!pages)
  1075. goto out;
  1076. for (i = 0; i < iov_count; i++) {
  1077. unsigned long uaddr = (unsigned long)iov[i].iov_base;
  1078. unsigned long len = iov[i].iov_len;
  1079. unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
  1080. unsigned long start = uaddr >> PAGE_SHIFT;
  1081. const int local_nr_pages = end - start;
  1082. const int page_limit = cur_page + local_nr_pages;
  1083. ret = get_user_pages_fast(uaddr, local_nr_pages,
  1084. write_to_vm, &pages[cur_page]);
  1085. if (ret < local_nr_pages) {
  1086. ret = -EFAULT;
  1087. goto out_unmap;
  1088. }
  1089. offset = uaddr & ~PAGE_MASK;
  1090. for (j = cur_page; j < page_limit; j++) {
  1091. unsigned int bytes = PAGE_SIZE - offset;
  1092. if (len <= 0)
  1093. break;
  1094. if (bytes > len)
  1095. bytes = len;
  1096. /*
  1097. * sorry...
  1098. */
  1099. if (bio_add_pc_page(q, bio, pages[j], bytes, offset) <
  1100. bytes)
  1101. break;
  1102. len -= bytes;
  1103. offset = 0;
  1104. }
  1105. cur_page = j;
  1106. /*
  1107. * release the pages we didn't map into the bio, if any
  1108. */
  1109. while (j < page_limit)
  1110. page_cache_release(pages[j++]);
  1111. }
  1112. kfree(pages);
  1113. /*
  1114. * set data direction, and check if mapped pages need bouncing
  1115. */
  1116. if (!write_to_vm)
  1117. bio->bi_rw |= REQ_WRITE;
  1118. bio->bi_bdev = bdev;
  1119. bio->bi_flags |= (1 << BIO_USER_MAPPED);
  1120. return bio;
  1121. out_unmap:
  1122. for (i = 0; i < nr_pages; i++) {
  1123. if(!pages[i])
  1124. break;
  1125. page_cache_release(pages[i]);
  1126. }
  1127. out:
  1128. kfree(pages);
  1129. bio_put(bio);
  1130. return ERR_PTR(ret);
  1131. }
  1132. /**
  1133. * bio_map_user - map user address into bio
  1134. * @q: the struct request_queue for the bio
  1135. * @bdev: destination block device
  1136. * @uaddr: start of user address
  1137. * @len: length in bytes
  1138. * @write_to_vm: bool indicating writing to pages or not
  1139. * @gfp_mask: memory allocation flags
  1140. *
  1141. * Map the user space address into a bio suitable for io to a block
  1142. * device. Returns an error pointer in case of error.
  1143. */
  1144. struct bio *bio_map_user(struct request_queue *q, struct block_device *bdev,
  1145. unsigned long uaddr, unsigned int len, int write_to_vm,
  1146. gfp_t gfp_mask)
  1147. {
  1148. struct sg_iovec iov;
  1149. iov.iov_base = (void __user *)uaddr;
  1150. iov.iov_len = len;
  1151. return bio_map_user_iov(q, bdev, &iov, 1, write_to_vm, gfp_mask);
  1152. }
  1153. EXPORT_SYMBOL(bio_map_user);
  1154. /**
  1155. * bio_map_user_iov - map user sg_iovec table into bio
  1156. * @q: the struct request_queue for the bio
  1157. * @bdev: destination block device
  1158. * @iov: the iovec.
  1159. * @iov_count: number of elements in the iovec
  1160. * @write_to_vm: bool indicating writing to pages or not
  1161. * @gfp_mask: memory allocation flags
  1162. *
  1163. * Map the user space address into a bio suitable for io to a block
  1164. * device. Returns an error pointer in case of error.
  1165. */
  1166. struct bio *bio_map_user_iov(struct request_queue *q, struct block_device *bdev,
  1167. struct sg_iovec *iov, int iov_count,
  1168. int write_to_vm, gfp_t gfp_mask)
  1169. {
  1170. struct bio *bio;
  1171. bio = __bio_map_user_iov(q, bdev, iov, iov_count, write_to_vm,
  1172. gfp_mask);
  1173. if (IS_ERR(bio))
  1174. return bio;
  1175. /*
  1176. * subtle -- if __bio_map_user() ended up bouncing a bio,
  1177. * it would normally disappear when its bi_end_io is run.
  1178. * however, we need it for the unmap, so grab an extra
  1179. * reference to it
  1180. */
  1181. bio_get(bio);
  1182. return bio;
  1183. }
  1184. static void __bio_unmap_user(struct bio *bio)
  1185. {
  1186. struct bio_vec *bvec;
  1187. int i;
  1188. /*
  1189. * make sure we dirty pages we wrote to
  1190. */
  1191. bio_for_each_segment_all(bvec, bio, i) {
  1192. if (bio_data_dir(bio) == READ)
  1193. set_page_dirty_lock(bvec->bv_page);
  1194. page_cache_release(bvec->bv_page);
  1195. }
  1196. bio_put(bio);
  1197. }
  1198. /**
  1199. * bio_unmap_user - unmap a bio
  1200. * @bio: the bio being unmapped
  1201. *
  1202. * Unmap a bio previously mapped by bio_map_user(). Must be called with
  1203. * a process context.
  1204. *
  1205. * bio_unmap_user() may sleep.
  1206. */
  1207. void bio_unmap_user(struct bio *bio)
  1208. {
  1209. __bio_unmap_user(bio);
  1210. bio_put(bio);
  1211. }
  1212. EXPORT_SYMBOL(bio_unmap_user);
  1213. static void bio_map_kern_endio(struct bio *bio, int err)
  1214. {
  1215. bio_put(bio);
  1216. }
  1217. static struct bio *__bio_map_kern(struct request_queue *q, void *data,
  1218. unsigned int len, gfp_t gfp_mask)
  1219. {
  1220. unsigned long kaddr = (unsigned long)data;
  1221. unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
  1222. unsigned long start = kaddr >> PAGE_SHIFT;
  1223. const int nr_pages = end - start;
  1224. int offset, i;
  1225. struct bio *bio;
  1226. bio = bio_kmalloc(gfp_mask, nr_pages);
  1227. if (!bio)
  1228. return ERR_PTR(-ENOMEM);
  1229. offset = offset_in_page(kaddr);
  1230. for (i = 0; i < nr_pages; i++) {
  1231. unsigned int bytes = PAGE_SIZE - offset;
  1232. if (len <= 0)
  1233. break;
  1234. if (bytes > len)
  1235. bytes = len;
  1236. if (bio_add_pc_page(q, bio, virt_to_page(data), bytes,
  1237. offset) < bytes)
  1238. break;
  1239. data += bytes;
  1240. len -= bytes;
  1241. offset = 0;
  1242. }
  1243. bio->bi_end_io = bio_map_kern_endio;
  1244. return bio;
  1245. }
  1246. /**
  1247. * bio_map_kern - map kernel address into bio
  1248. * @q: the struct request_queue for the bio
  1249. * @data: pointer to buffer to map
  1250. * @len: length in bytes
  1251. * @gfp_mask: allocation flags for bio allocation
  1252. *
  1253. * Map the kernel address into a bio suitable for io to a block
  1254. * device. Returns an error pointer in case of error.
  1255. */
  1256. struct bio *bio_map_kern(struct request_queue *q, void *data, unsigned int len,
  1257. gfp_t gfp_mask)
  1258. {
  1259. struct bio *bio;
  1260. bio = __bio_map_kern(q, data, len, gfp_mask);
  1261. if (IS_ERR(bio))
  1262. return bio;
  1263. if (bio->bi_size == len)
  1264. return bio;
  1265. /*
  1266. * Don't support partial mappings.
  1267. */
  1268. bio_put(bio);
  1269. return ERR_PTR(-EINVAL);
  1270. }
  1271. EXPORT_SYMBOL(bio_map_kern);
  1272. static void bio_copy_kern_endio(struct bio *bio, int err)
  1273. {
  1274. struct bio_vec *bvec;
  1275. const int read = bio_data_dir(bio) == READ;
  1276. struct bio_map_data *bmd = bio->bi_private;
  1277. int i;
  1278. char *p = bmd->sgvecs[0].iov_base;
  1279. bio_for_each_segment_all(bvec, bio, i) {
  1280. char *addr = page_address(bvec->bv_page);
  1281. int len = bmd->iovecs[i].bv_len;
  1282. if (read)
  1283. memcpy(p, addr, len);
  1284. __free_page(bvec->bv_page);
  1285. p += len;
  1286. }
  1287. bio_free_map_data(bmd);
  1288. bio_put(bio);
  1289. }
  1290. /**
  1291. * bio_copy_kern - copy kernel address into bio
  1292. * @q: the struct request_queue for the bio
  1293. * @data: pointer to buffer to copy
  1294. * @len: length in bytes
  1295. * @gfp_mask: allocation flags for bio and page allocation
  1296. * @reading: data direction is READ
  1297. *
  1298. * copy the kernel address into a bio suitable for io to a block
  1299. * device. Returns an error pointer in case of error.
  1300. */
  1301. struct bio *bio_copy_kern(struct request_queue *q, void *data, unsigned int len,
  1302. gfp_t gfp_mask, int reading)
  1303. {
  1304. struct bio *bio;
  1305. struct bio_vec *bvec;
  1306. int i;
  1307. bio = bio_copy_user(q, NULL, (unsigned long)data, len, 1, gfp_mask);
  1308. if (IS_ERR(bio))
  1309. return bio;
  1310. if (!reading) {
  1311. void *p = data;
  1312. bio_for_each_segment_all(bvec, bio, i) {
  1313. char *addr = page_address(bvec->bv_page);
  1314. memcpy(addr, p, bvec->bv_len);
  1315. p += bvec->bv_len;
  1316. }
  1317. }
  1318. bio->bi_end_io = bio_copy_kern_endio;
  1319. return bio;
  1320. }
  1321. EXPORT_SYMBOL(bio_copy_kern);
  1322. /*
  1323. * bio_set_pages_dirty() and bio_check_pages_dirty() are support functions
  1324. * for performing direct-IO in BIOs.
  1325. *
  1326. * The problem is that we cannot run set_page_dirty() from interrupt context
  1327. * because the required locks are not interrupt-safe. So what we can do is to
  1328. * mark the pages dirty _before_ performing IO. And in interrupt context,
  1329. * check that the pages are still dirty. If so, fine. If not, redirty them
  1330. * in process context.
  1331. *
  1332. * We special-case compound pages here: normally this means reads into hugetlb
  1333. * pages. The logic in here doesn't really work right for compound pages
  1334. * because the VM does not uniformly chase down the head page in all cases.
  1335. * But dirtiness of compound pages is pretty meaningless anyway: the VM doesn't
  1336. * handle them at all. So we skip compound pages here at an early stage.
  1337. *
  1338. * Note that this code is very hard to test under normal circumstances because
  1339. * direct-io pins the pages with get_user_pages(). This makes
  1340. * is_page_cache_freeable return false, and the VM will not clean the pages.
  1341. * But other code (eg, flusher threads) could clean the pages if they are mapped
  1342. * pagecache.
  1343. *
  1344. * Simply disabling the call to bio_set_pages_dirty() is a good way to test the
  1345. * deferred bio dirtying paths.
  1346. */
  1347. /*
  1348. * bio_set_pages_dirty() will mark all the bio's pages as dirty.
  1349. */
  1350. void bio_set_pages_dirty(struct bio *bio)
  1351. {
  1352. struct bio_vec *bvec;
  1353. int i;
  1354. bio_for_each_segment_all(bvec, bio, i) {
  1355. struct page *page = bvec->bv_page;
  1356. if (page && !PageCompound(page))
  1357. set_page_dirty_lock(page);
  1358. }
  1359. }
  1360. static void bio_release_pages(struct bio *bio)
  1361. {
  1362. struct bio_vec *bvec;
  1363. int i;
  1364. bio_for_each_segment_all(bvec, bio, i) {
  1365. struct page *page = bvec->bv_page;
  1366. if (page)
  1367. put_page(page);
  1368. }
  1369. }
  1370. /*
  1371. * bio_check_pages_dirty() will check that all the BIO's pages are still dirty.
  1372. * If they are, then fine. If, however, some pages are clean then they must
  1373. * have been written out during the direct-IO read. So we take another ref on
  1374. * the BIO and the offending pages and re-dirty the pages in process context.
  1375. *
  1376. * It is expected that bio_check_pages_dirty() will wholly own the BIO from
  1377. * here on. It will run one page_cache_release() against each page and will
  1378. * run one bio_put() against the BIO.
  1379. */
  1380. static void bio_dirty_fn(struct work_struct *work);
  1381. static DECLARE_WORK(bio_dirty_work, bio_dirty_fn);
  1382. static DEFINE_SPINLOCK(bio_dirty_lock);
  1383. static struct bio *bio_dirty_list;
  1384. /*
  1385. * This runs in process context
  1386. */
  1387. static void bio_dirty_fn(struct work_struct *work)
  1388. {
  1389. unsigned long flags;
  1390. struct bio *bio;
  1391. spin_lock_irqsave(&bio_dirty_lock, flags);
  1392. bio = bio_dirty_list;
  1393. bio_dirty_list = NULL;
  1394. spin_unlock_irqrestore(&bio_dirty_lock, flags);
  1395. while (bio) {
  1396. struct bio *next = bio->bi_private;
  1397. bio_set_pages_dirty(bio);
  1398. bio_release_pages(bio);
  1399. bio_put(bio);
  1400. bio = next;
  1401. }
  1402. }
  1403. void bio_check_pages_dirty(struct bio *bio)
  1404. {
  1405. struct bio_vec *bvec;
  1406. int nr_clean_pages = 0;
  1407. int i;
  1408. bio_for_each_segment_all(bvec, bio, i) {
  1409. struct page *page = bvec->bv_page;
  1410. if (PageDirty(page) || PageCompound(page)) {
  1411. page_cache_release(page);
  1412. bvec->bv_page = NULL;
  1413. } else {
  1414. nr_clean_pages++;
  1415. }
  1416. }
  1417. if (nr_clean_pages) {
  1418. unsigned long flags;
  1419. spin_lock_irqsave(&bio_dirty_lock, flags);
  1420. bio->bi_private = bio_dirty_list;
  1421. bio_dirty_list = bio;
  1422. spin_unlock_irqrestore(&bio_dirty_lock, flags);
  1423. schedule_work(&bio_dirty_work);
  1424. } else {
  1425. bio_put(bio);
  1426. }
  1427. }
  1428. #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
  1429. void bio_flush_dcache_pages(struct bio *bi)
  1430. {
  1431. int i;
  1432. struct bio_vec *bvec;
  1433. bio_for_each_segment(bvec, bi, i)
  1434. flush_dcache_page(bvec->bv_page);
  1435. }
  1436. EXPORT_SYMBOL(bio_flush_dcache_pages);
  1437. #endif
  1438. /**
  1439. * bio_endio - end I/O on a bio
  1440. * @bio: bio
  1441. * @error: error, if any
  1442. *
  1443. * Description:
  1444. * bio_endio() will end I/O on the whole bio. bio_endio() is the
  1445. * preferred way to end I/O on a bio, it takes care of clearing
  1446. * BIO_UPTODATE on error. @error is 0 on success, and and one of the
  1447. * established -Exxxx (-EIO, for instance) error values in case
  1448. * something went wrong. No one should call bi_end_io() directly on a
  1449. * bio unless they own it and thus know that it has an end_io
  1450. * function.
  1451. **/
  1452. void bio_endio(struct bio *bio, int error)
  1453. {
  1454. if (error)
  1455. clear_bit(BIO_UPTODATE, &bio->bi_flags);
  1456. else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
  1457. error = -EIO;
  1458. if (bio->bi_end_io)
  1459. bio->bi_end_io(bio, error);
  1460. }
  1461. EXPORT_SYMBOL(bio_endio);
  1462. void bio_pair_release(struct bio_pair *bp)
  1463. {
  1464. if (atomic_dec_and_test(&bp->cnt)) {
  1465. struct bio *master = bp->bio1.bi_private;
  1466. bio_endio(master, bp->error);
  1467. mempool_free(bp, bp->bio2.bi_private);
  1468. }
  1469. }
  1470. EXPORT_SYMBOL(bio_pair_release);
  1471. static void bio_pair_end_1(struct bio *bi, int err)
  1472. {
  1473. struct bio_pair *bp = container_of(bi, struct bio_pair, bio1);
  1474. if (err)
  1475. bp->error = err;
  1476. bio_pair_release(bp);
  1477. }
  1478. static void bio_pair_end_2(struct bio *bi, int err)
  1479. {
  1480. struct bio_pair *bp = container_of(bi, struct bio_pair, bio2);
  1481. if (err)
  1482. bp->error = err;
  1483. bio_pair_release(bp);
  1484. }
  1485. /*
  1486. * split a bio - only worry about a bio with a single page in its iovec
  1487. */
  1488. struct bio_pair *bio_split(struct bio *bi, int first_sectors)
  1489. {
  1490. struct bio_pair *bp = mempool_alloc(bio_split_pool, GFP_NOIO);
  1491. if (!bp)
  1492. return bp;
  1493. trace_block_split(bdev_get_queue(bi->bi_bdev), bi,
  1494. bi->bi_sector + first_sectors);
  1495. BUG_ON(bio_segments(bi) > 1);
  1496. atomic_set(&bp->cnt, 3);
  1497. bp->error = 0;
  1498. bp->bio1 = *bi;
  1499. bp->bio2 = *bi;
  1500. bp->bio2.bi_sector += first_sectors;
  1501. bp->bio2.bi_size -= first_sectors << 9;
  1502. bp->bio1.bi_size = first_sectors << 9;
  1503. if (bi->bi_vcnt != 0) {
  1504. bp->bv1 = *bio_iovec(bi);
  1505. bp->bv2 = *bio_iovec(bi);
  1506. if (bio_is_rw(bi)) {
  1507. bp->bv2.bv_offset += first_sectors << 9;
  1508. bp->bv2.bv_len -= first_sectors << 9;
  1509. bp->bv1.bv_len = first_sectors << 9;
  1510. }
  1511. bp->bio1.bi_io_vec = &bp->bv1;
  1512. bp->bio2.bi_io_vec = &bp->bv2;
  1513. bp->bio1.bi_max_vecs = 1;
  1514. bp->bio2.bi_max_vecs = 1;
  1515. }
  1516. bp->bio1.bi_end_io = bio_pair_end_1;
  1517. bp->bio2.bi_end_io = bio_pair_end_2;
  1518. bp->bio1.bi_private = bi;
  1519. bp->bio2.bi_private = bio_split_pool;
  1520. if (bio_integrity(bi))
  1521. bio_integrity_split(bi, bp, first_sectors);
  1522. return bp;
  1523. }
  1524. EXPORT_SYMBOL(bio_split);
  1525. /**
  1526. * bio_sector_offset - Find hardware sector offset in bio
  1527. * @bio: bio to inspect
  1528. * @index: bio_vec index
  1529. * @offset: offset in bv_page
  1530. *
  1531. * Return the number of hardware sectors between beginning of bio
  1532. * and an end point indicated by a bio_vec index and an offset
  1533. * within that vector's page.
  1534. */
  1535. sector_t bio_sector_offset(struct bio *bio, unsigned short index,
  1536. unsigned int offset)
  1537. {
  1538. unsigned int sector_sz;
  1539. struct bio_vec *bv;
  1540. sector_t sectors;
  1541. int i;
  1542. sector_sz = queue_logical_block_size(bio->bi_bdev->bd_disk->queue);
  1543. sectors = 0;
  1544. if (index >= bio->bi_idx)
  1545. index = bio->bi_vcnt - 1;
  1546. bio_for_each_segment_all(bv, bio, i) {
  1547. if (i == index) {
  1548. if (offset > bv->bv_offset)
  1549. sectors += (offset - bv->bv_offset) / sector_sz;
  1550. break;
  1551. }
  1552. sectors += bv->bv_len / sector_sz;
  1553. }
  1554. return sectors;
  1555. }
  1556. EXPORT_SYMBOL(bio_sector_offset);
  1557. /*
  1558. * create memory pools for biovec's in a bio_set.
  1559. * use the global biovec slabs created for general use.
  1560. */
  1561. mempool_t *biovec_create_pool(struct bio_set *bs, int pool_entries)
  1562. {
  1563. struct biovec_slab *bp = bvec_slabs + BIOVEC_MAX_IDX;
  1564. return mempool_create_slab_pool(pool_entries, bp->slab);
  1565. }
  1566. void bioset_free(struct bio_set *bs)
  1567. {
  1568. if (bs->rescue_workqueue)
  1569. destroy_workqueue(bs->rescue_workqueue);
  1570. if (bs->bio_pool)
  1571. mempool_destroy(bs->bio_pool);
  1572. if (bs->bvec_pool)
  1573. mempool_destroy(bs->bvec_pool);
  1574. bioset_integrity_free(bs);
  1575. bio_put_slab(bs);
  1576. kfree(bs);
  1577. }
  1578. EXPORT_SYMBOL(bioset_free);
  1579. /**
  1580. * bioset_create - Create a bio_set
  1581. * @pool_size: Number of bio and bio_vecs to cache in the mempool
  1582. * @front_pad: Number of bytes to allocate in front of the returned bio
  1583. *
  1584. * Description:
  1585. * Set up a bio_set to be used with @bio_alloc_bioset. Allows the caller
  1586. * to ask for a number of bytes to be allocated in front of the bio.
  1587. * Front pad allocation is useful for embedding the bio inside
  1588. * another structure, to avoid allocating extra data to go with the bio.
  1589. * Note that the bio must be embedded at the END of that structure always,
  1590. * or things will break badly.
  1591. */
  1592. struct bio_set *bioset_create(unsigned int pool_size, unsigned int front_pad)
  1593. {
  1594. unsigned int back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec);
  1595. struct bio_set *bs;
  1596. bs = kzalloc(sizeof(*bs), GFP_KERNEL);
  1597. if (!bs)
  1598. return NULL;
  1599. bs->front_pad = front_pad;
  1600. spin_lock_init(&bs->rescue_lock);
  1601. bio_list_init(&bs->rescue_list);
  1602. INIT_WORK(&bs->rescue_work, bio_alloc_rescue);
  1603. bs->bio_slab = bio_find_or_create_slab(front_pad + back_pad);
  1604. if (!bs->bio_slab) {
  1605. kfree(bs);
  1606. return NULL;
  1607. }
  1608. bs->bio_pool = mempool_create_slab_pool(pool_size, bs->bio_slab);
  1609. if (!bs->bio_pool)
  1610. goto bad;
  1611. bs->bvec_pool = biovec_create_pool(bs, pool_size);
  1612. if (!bs->bvec_pool)
  1613. goto bad;
  1614. bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0);
  1615. if (!bs->rescue_workqueue)
  1616. goto bad;
  1617. return bs;
  1618. bad:
  1619. bioset_free(bs);
  1620. return NULL;
  1621. }
  1622. EXPORT_SYMBOL(bioset_create);
  1623. #ifdef CONFIG_BLK_CGROUP
  1624. /**
  1625. * bio_associate_current - associate a bio with %current
  1626. * @bio: target bio
  1627. *
  1628. * Associate @bio with %current if it hasn't been associated yet. Block
  1629. * layer will treat @bio as if it were issued by %current no matter which
  1630. * task actually issues it.
  1631. *
  1632. * This function takes an extra reference of @task's io_context and blkcg
  1633. * which will be put when @bio is released. The caller must own @bio,
  1634. * ensure %current->io_context exists, and is responsible for synchronizing
  1635. * calls to this function.
  1636. */
  1637. int bio_associate_current(struct bio *bio)
  1638. {
  1639. struct io_context *ioc;
  1640. struct cgroup_subsys_state *css;
  1641. if (bio->bi_ioc)
  1642. return -EBUSY;
  1643. ioc = current->io_context;
  1644. if (!ioc)
  1645. return -ENOENT;
  1646. /* acquire active ref on @ioc and associate */
  1647. get_io_context_active(ioc);
  1648. bio->bi_ioc = ioc;
  1649. /* associate blkcg if exists */
  1650. rcu_read_lock();
  1651. css = task_css(current, blkio_subsys_id);
  1652. if (css && css_tryget(css))
  1653. bio->bi_css = css;
  1654. rcu_read_unlock();
  1655. return 0;
  1656. }
  1657. /**
  1658. * bio_disassociate_task - undo bio_associate_current()
  1659. * @bio: target bio
  1660. */
  1661. void bio_disassociate_task(struct bio *bio)
  1662. {
  1663. if (bio->bi_ioc) {
  1664. put_io_context(bio->bi_ioc);
  1665. bio->bi_ioc = NULL;
  1666. }
  1667. if (bio->bi_css) {
  1668. css_put(bio->bi_css);
  1669. bio->bi_css = NULL;
  1670. }
  1671. }
  1672. #endif /* CONFIG_BLK_CGROUP */
  1673. static void __init biovec_init_slabs(void)
  1674. {
  1675. int i;
  1676. for (i = 0; i < BIOVEC_NR_POOLS; i++) {
  1677. int size;
  1678. struct biovec_slab *bvs = bvec_slabs + i;
  1679. if (bvs->nr_vecs <= BIO_INLINE_VECS) {
  1680. bvs->slab = NULL;
  1681. continue;
  1682. }
  1683. size = bvs->nr_vecs * sizeof(struct bio_vec);
  1684. bvs->slab = kmem_cache_create(bvs->name, size, 0,
  1685. SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
  1686. }
  1687. }
  1688. static int __init init_bio(void)
  1689. {
  1690. bio_slab_max = 2;
  1691. bio_slab_nr = 0;
  1692. bio_slabs = kzalloc(bio_slab_max * sizeof(struct bio_slab), GFP_KERNEL);
  1693. if (!bio_slabs)
  1694. panic("bio: can't allocate bios\n");
  1695. bio_integrity_init();
  1696. biovec_init_slabs();
  1697. fs_bio_set = bioset_create(BIO_POOL_SIZE, 0);
  1698. if (!fs_bio_set)
  1699. panic("bio: can't allocate bios\n");
  1700. if (bioset_integrity_create(fs_bio_set, BIO_POOL_SIZE))
  1701. panic("bio: can't create integrity pool\n");
  1702. bio_split_pool = mempool_create_kmalloc_pool(BIO_SPLIT_ENTRIES,
  1703. sizeof(struct bio_pair));
  1704. if (!bio_split_pool)
  1705. panic("bio: can't create split pool\n");
  1706. return 0;
  1707. }
  1708. subsys_initcall(init_bio);