dm.c 35 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771
  1. /*
  2. * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
  3. * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
  4. *
  5. * This file is released under the GPL.
  6. */
  7. #include "dm.h"
  8. #include "dm-bio-list.h"
  9. #include "dm-uevent.h"
  10. #include <linux/init.h>
  11. #include <linux/module.h>
  12. #include <linux/mutex.h>
  13. #include <linux/moduleparam.h>
  14. #include <linux/blkpg.h>
  15. #include <linux/bio.h>
  16. #include <linux/buffer_head.h>
  17. #include <linux/mempool.h>
  18. #include <linux/slab.h>
  19. #include <linux/idr.h>
  20. #include <linux/hdreg.h>
  21. #include <linux/blktrace_api.h>
  22. #include <trace/block.h>
  23. #define DM_MSG_PREFIX "core"
  24. static const char *_name = DM_NAME;
  25. static unsigned int major = 0;
  26. static unsigned int _major = 0;
  27. static DEFINE_SPINLOCK(_minor_lock);
  28. /*
  29. * For bio-based dm.
  30. * One of these is allocated per bio.
  31. */
  32. struct dm_io {
  33. struct mapped_device *md;
  34. int error;
  35. atomic_t io_count;
  36. struct bio *bio;
  37. unsigned long start_time;
  38. };
  39. /*
  40. * For bio-based dm.
  41. * One of these is allocated per target within a bio. Hopefully
  42. * this will be simplified out one day.
  43. */
  44. struct dm_target_io {
  45. struct dm_io *io;
  46. struct dm_target *ti;
  47. union map_info info;
  48. };
  49. DEFINE_TRACE(block_bio_complete);
  50. /*
  51. * For request-based dm.
  52. * One of these is allocated per request.
  53. */
  54. struct dm_rq_target_io {
  55. struct mapped_device *md;
  56. struct dm_target *ti;
  57. struct request *orig, clone;
  58. int error;
  59. union map_info info;
  60. };
  61. /*
  62. * For request-based dm.
  63. * One of these is allocated per bio.
  64. */
  65. struct dm_rq_clone_bio_info {
  66. struct bio *orig;
  67. struct request *rq;
  68. };
  69. union map_info *dm_get_mapinfo(struct bio *bio)
  70. {
  71. if (bio && bio->bi_private)
  72. return &((struct dm_target_io *)bio->bi_private)->info;
  73. return NULL;
  74. }
  75. #define MINOR_ALLOCED ((void *)-1)
  76. /*
  77. * Bits for the md->flags field.
  78. */
  79. #define DMF_BLOCK_IO 0
  80. #define DMF_SUSPENDED 1
  81. #define DMF_FROZEN 2
  82. #define DMF_FREEING 3
  83. #define DMF_DELETING 4
  84. #define DMF_NOFLUSH_SUSPENDING 5
  85. /*
  86. * Work processed by per-device workqueue.
  87. */
  88. struct dm_wq_req {
  89. enum {
  90. DM_WQ_FLUSH_DEFERRED,
  91. } type;
  92. struct work_struct work;
  93. struct mapped_device *md;
  94. void *context;
  95. };
  96. struct mapped_device {
  97. struct rw_semaphore io_lock;
  98. struct mutex suspend_lock;
  99. spinlock_t pushback_lock;
  100. rwlock_t map_lock;
  101. atomic_t holders;
  102. atomic_t open_count;
  103. unsigned long flags;
  104. struct request_queue *queue;
  105. struct gendisk *disk;
  106. char name[16];
  107. void *interface_ptr;
  108. /*
  109. * A list of ios that arrived while we were suspended.
  110. */
  111. atomic_t pending;
  112. wait_queue_head_t wait;
  113. struct bio_list deferred;
  114. struct bio_list pushback;
  115. /*
  116. * Processing queue (flush/barriers)
  117. */
  118. struct workqueue_struct *wq;
  119. /*
  120. * The current mapping.
  121. */
  122. struct dm_table *map;
  123. /*
  124. * io objects are allocated from here.
  125. */
  126. mempool_t *io_pool;
  127. mempool_t *tio_pool;
  128. struct bio_set *bs;
  129. /*
  130. * Event handling.
  131. */
  132. atomic_t event_nr;
  133. wait_queue_head_t eventq;
  134. atomic_t uevent_seq;
  135. struct list_head uevent_list;
  136. spinlock_t uevent_lock; /* Protect access to uevent_list */
  137. /*
  138. * freeze/thaw support require holding onto a super block
  139. */
  140. struct super_block *frozen_sb;
  141. struct block_device *suspended_bdev;
  142. /* forced geometry settings */
  143. struct hd_geometry geometry;
  144. /* sysfs handle */
  145. struct kobject kobj;
  146. };
  147. #define MIN_IOS 256
  148. static struct kmem_cache *_io_cache;
  149. static struct kmem_cache *_tio_cache;
  150. static struct kmem_cache *_rq_tio_cache;
  151. static struct kmem_cache *_rq_bio_info_cache;
  152. static int __init local_init(void)
  153. {
  154. int r = -ENOMEM;
  155. /* allocate a slab for the dm_ios */
  156. _io_cache = KMEM_CACHE(dm_io, 0);
  157. if (!_io_cache)
  158. return r;
  159. /* allocate a slab for the target ios */
  160. _tio_cache = KMEM_CACHE(dm_target_io, 0);
  161. if (!_tio_cache)
  162. goto out_free_io_cache;
  163. _rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0);
  164. if (!_rq_tio_cache)
  165. goto out_free_tio_cache;
  166. _rq_bio_info_cache = KMEM_CACHE(dm_rq_clone_bio_info, 0);
  167. if (!_rq_bio_info_cache)
  168. goto out_free_rq_tio_cache;
  169. r = dm_uevent_init();
  170. if (r)
  171. goto out_free_rq_bio_info_cache;
  172. _major = major;
  173. r = register_blkdev(_major, _name);
  174. if (r < 0)
  175. goto out_uevent_exit;
  176. if (!_major)
  177. _major = r;
  178. return 0;
  179. out_uevent_exit:
  180. dm_uevent_exit();
  181. out_free_rq_bio_info_cache:
  182. kmem_cache_destroy(_rq_bio_info_cache);
  183. out_free_rq_tio_cache:
  184. kmem_cache_destroy(_rq_tio_cache);
  185. out_free_tio_cache:
  186. kmem_cache_destroy(_tio_cache);
  187. out_free_io_cache:
  188. kmem_cache_destroy(_io_cache);
  189. return r;
  190. }
  191. static void local_exit(void)
  192. {
  193. kmem_cache_destroy(_rq_bio_info_cache);
  194. kmem_cache_destroy(_rq_tio_cache);
  195. kmem_cache_destroy(_tio_cache);
  196. kmem_cache_destroy(_io_cache);
  197. unregister_blkdev(_major, _name);
  198. dm_uevent_exit();
  199. _major = 0;
  200. DMINFO("cleaned up");
  201. }
  202. static int (*_inits[])(void) __initdata = {
  203. local_init,
  204. dm_target_init,
  205. dm_linear_init,
  206. dm_stripe_init,
  207. dm_kcopyd_init,
  208. dm_interface_init,
  209. };
  210. static void (*_exits[])(void) = {
  211. local_exit,
  212. dm_target_exit,
  213. dm_linear_exit,
  214. dm_stripe_exit,
  215. dm_kcopyd_exit,
  216. dm_interface_exit,
  217. };
  218. static int __init dm_init(void)
  219. {
  220. const int count = ARRAY_SIZE(_inits);
  221. int r, i;
  222. for (i = 0; i < count; i++) {
  223. r = _inits[i]();
  224. if (r)
  225. goto bad;
  226. }
  227. return 0;
  228. bad:
  229. while (i--)
  230. _exits[i]();
  231. return r;
  232. }
  233. static void __exit dm_exit(void)
  234. {
  235. int i = ARRAY_SIZE(_exits);
  236. while (i--)
  237. _exits[i]();
  238. }
  239. /*
  240. * Block device functions
  241. */
  242. static int dm_blk_open(struct block_device *bdev, fmode_t mode)
  243. {
  244. struct mapped_device *md;
  245. spin_lock(&_minor_lock);
  246. md = bdev->bd_disk->private_data;
  247. if (!md)
  248. goto out;
  249. if (test_bit(DMF_FREEING, &md->flags) ||
  250. test_bit(DMF_DELETING, &md->flags)) {
  251. md = NULL;
  252. goto out;
  253. }
  254. dm_get(md);
  255. atomic_inc(&md->open_count);
  256. out:
  257. spin_unlock(&_minor_lock);
  258. return md ? 0 : -ENXIO;
  259. }
  260. static int dm_blk_close(struct gendisk *disk, fmode_t mode)
  261. {
  262. struct mapped_device *md = disk->private_data;
  263. atomic_dec(&md->open_count);
  264. dm_put(md);
  265. return 0;
  266. }
  267. int dm_open_count(struct mapped_device *md)
  268. {
  269. return atomic_read(&md->open_count);
  270. }
  271. /*
  272. * Guarantees nothing is using the device before it's deleted.
  273. */
  274. int dm_lock_for_deletion(struct mapped_device *md)
  275. {
  276. int r = 0;
  277. spin_lock(&_minor_lock);
  278. if (dm_open_count(md))
  279. r = -EBUSY;
  280. else
  281. set_bit(DMF_DELETING, &md->flags);
  282. spin_unlock(&_minor_lock);
  283. return r;
  284. }
  285. static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
  286. {
  287. struct mapped_device *md = bdev->bd_disk->private_data;
  288. return dm_get_geometry(md, geo);
  289. }
  290. static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
  291. unsigned int cmd, unsigned long arg)
  292. {
  293. struct mapped_device *md = bdev->bd_disk->private_data;
  294. struct dm_table *map = dm_get_table(md);
  295. struct dm_target *tgt;
  296. int r = -ENOTTY;
  297. if (!map || !dm_table_get_size(map))
  298. goto out;
  299. /* We only support devices that have a single target */
  300. if (dm_table_get_num_targets(map) != 1)
  301. goto out;
  302. tgt = dm_table_get_target(map, 0);
  303. if (dm_suspended(md)) {
  304. r = -EAGAIN;
  305. goto out;
  306. }
  307. if (tgt->type->ioctl)
  308. r = tgt->type->ioctl(tgt, cmd, arg);
  309. out:
  310. dm_table_put(map);
  311. return r;
  312. }
  313. static struct dm_io *alloc_io(struct mapped_device *md)
  314. {
  315. return mempool_alloc(md->io_pool, GFP_NOIO);
  316. }
  317. static void free_io(struct mapped_device *md, struct dm_io *io)
  318. {
  319. mempool_free(io, md->io_pool);
  320. }
  321. static struct dm_target_io *alloc_tio(struct mapped_device *md)
  322. {
  323. return mempool_alloc(md->tio_pool, GFP_NOIO);
  324. }
  325. static void free_tio(struct mapped_device *md, struct dm_target_io *tio)
  326. {
  327. mempool_free(tio, md->tio_pool);
  328. }
  329. static void start_io_acct(struct dm_io *io)
  330. {
  331. struct mapped_device *md = io->md;
  332. int cpu;
  333. io->start_time = jiffies;
  334. cpu = part_stat_lock();
  335. part_round_stats(cpu, &dm_disk(md)->part0);
  336. part_stat_unlock();
  337. dm_disk(md)->part0.in_flight = atomic_inc_return(&md->pending);
  338. }
  339. static void end_io_acct(struct dm_io *io)
  340. {
  341. struct mapped_device *md = io->md;
  342. struct bio *bio = io->bio;
  343. unsigned long duration = jiffies - io->start_time;
  344. int pending, cpu;
  345. int rw = bio_data_dir(bio);
  346. cpu = part_stat_lock();
  347. part_round_stats(cpu, &dm_disk(md)->part0);
  348. part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration);
  349. part_stat_unlock();
  350. dm_disk(md)->part0.in_flight = pending =
  351. atomic_dec_return(&md->pending);
  352. /* nudge anyone waiting on suspend queue */
  353. if (!pending)
  354. wake_up(&md->wait);
  355. }
  356. /*
  357. * Add the bio to the list of deferred io.
  358. */
  359. static int queue_io(struct mapped_device *md, struct bio *bio)
  360. {
  361. down_write(&md->io_lock);
  362. if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
  363. up_write(&md->io_lock);
  364. return 1;
  365. }
  366. bio_list_add(&md->deferred, bio);
  367. up_write(&md->io_lock);
  368. return 0; /* deferred successfully */
  369. }
  370. /*
  371. * Everyone (including functions in this file), should use this
  372. * function to access the md->map field, and make sure they call
  373. * dm_table_put() when finished.
  374. */
  375. struct dm_table *dm_get_table(struct mapped_device *md)
  376. {
  377. struct dm_table *t;
  378. read_lock(&md->map_lock);
  379. t = md->map;
  380. if (t)
  381. dm_table_get(t);
  382. read_unlock(&md->map_lock);
  383. return t;
  384. }
  385. /*
  386. * Get the geometry associated with a dm device
  387. */
  388. int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
  389. {
  390. *geo = md->geometry;
  391. return 0;
  392. }
  393. /*
  394. * Set the geometry of a device.
  395. */
  396. int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
  397. {
  398. sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
  399. if (geo->start > sz) {
  400. DMWARN("Start sector is beyond the geometry limits.");
  401. return -EINVAL;
  402. }
  403. md->geometry = *geo;
  404. return 0;
  405. }
  406. /*-----------------------------------------------------------------
  407. * CRUD START:
  408. * A more elegant soln is in the works that uses the queue
  409. * merge fn, unfortunately there are a couple of changes to
  410. * the block layer that I want to make for this. So in the
  411. * interests of getting something for people to use I give
  412. * you this clearly demarcated crap.
  413. *---------------------------------------------------------------*/
  414. static int __noflush_suspending(struct mapped_device *md)
  415. {
  416. return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
  417. }
  418. /*
  419. * Decrements the number of outstanding ios that a bio has been
  420. * cloned into, completing the original io if necc.
  421. */
  422. static void dec_pending(struct dm_io *io, int error)
  423. {
  424. unsigned long flags;
  425. int io_error;
  426. struct bio *bio;
  427. struct mapped_device *md = io->md;
  428. /* Push-back supersedes any I/O errors */
  429. if (error && !(io->error > 0 && __noflush_suspending(md)))
  430. io->error = error;
  431. if (atomic_dec_and_test(&io->io_count)) {
  432. if (io->error == DM_ENDIO_REQUEUE) {
  433. /*
  434. * Target requested pushing back the I/O.
  435. * This must be handled before the sleeper on
  436. * suspend queue merges the pushback list.
  437. */
  438. spin_lock_irqsave(&md->pushback_lock, flags);
  439. if (__noflush_suspending(md))
  440. bio_list_add(&md->pushback, io->bio);
  441. else
  442. /* noflush suspend was interrupted. */
  443. io->error = -EIO;
  444. spin_unlock_irqrestore(&md->pushback_lock, flags);
  445. }
  446. end_io_acct(io);
  447. io_error = io->error;
  448. bio = io->bio;
  449. free_io(md, io);
  450. if (io_error != DM_ENDIO_REQUEUE) {
  451. trace_block_bio_complete(md->queue, bio);
  452. bio_endio(bio, io_error);
  453. }
  454. }
  455. }
  456. static void clone_endio(struct bio *bio, int error)
  457. {
  458. int r = 0;
  459. struct dm_target_io *tio = bio->bi_private;
  460. struct dm_io *io = tio->io;
  461. struct mapped_device *md = tio->io->md;
  462. dm_endio_fn endio = tio->ti->type->end_io;
  463. if (!bio_flagged(bio, BIO_UPTODATE) && !error)
  464. error = -EIO;
  465. if (endio) {
  466. r = endio(tio->ti, bio, error, &tio->info);
  467. if (r < 0 || r == DM_ENDIO_REQUEUE)
  468. /*
  469. * error and requeue request are handled
  470. * in dec_pending().
  471. */
  472. error = r;
  473. else if (r == DM_ENDIO_INCOMPLETE)
  474. /* The target will handle the io */
  475. return;
  476. else if (r) {
  477. DMWARN("unimplemented target endio return value: %d", r);
  478. BUG();
  479. }
  480. }
  481. /*
  482. * Store md for cleanup instead of tio which is about to get freed.
  483. */
  484. bio->bi_private = md->bs;
  485. free_tio(md, tio);
  486. bio_put(bio);
  487. dec_pending(io, error);
  488. }
  489. static sector_t max_io_len(struct mapped_device *md,
  490. sector_t sector, struct dm_target *ti)
  491. {
  492. sector_t offset = sector - ti->begin;
  493. sector_t len = ti->len - offset;
  494. /*
  495. * Does the target need to split even further ?
  496. */
  497. if (ti->split_io) {
  498. sector_t boundary;
  499. boundary = ((offset + ti->split_io) & ~(ti->split_io - 1))
  500. - offset;
  501. if (len > boundary)
  502. len = boundary;
  503. }
  504. return len;
  505. }
  506. static void __map_bio(struct dm_target *ti, struct bio *clone,
  507. struct dm_target_io *tio)
  508. {
  509. int r;
  510. sector_t sector;
  511. struct mapped_device *md;
  512. /*
  513. * Sanity checks.
  514. */
  515. BUG_ON(!clone->bi_size);
  516. clone->bi_end_io = clone_endio;
  517. clone->bi_private = tio;
  518. /*
  519. * Map the clone. If r == 0 we don't need to do
  520. * anything, the target has assumed ownership of
  521. * this io.
  522. */
  523. atomic_inc(&tio->io->io_count);
  524. sector = clone->bi_sector;
  525. r = ti->type->map(ti, clone, &tio->info);
  526. if (r == DM_MAPIO_REMAPPED) {
  527. /* the bio has been remapped so dispatch it */
  528. trace_block_remap(bdev_get_queue(clone->bi_bdev), clone,
  529. tio->io->bio->bi_bdev->bd_dev,
  530. clone->bi_sector, sector);
  531. generic_make_request(clone);
  532. } else if (r < 0 || r == DM_MAPIO_REQUEUE) {
  533. /* error the io and bail out, or requeue it if needed */
  534. md = tio->io->md;
  535. dec_pending(tio->io, r);
  536. /*
  537. * Store bio_set for cleanup.
  538. */
  539. clone->bi_private = md->bs;
  540. bio_put(clone);
  541. free_tio(md, tio);
  542. } else if (r) {
  543. DMWARN("unimplemented target map return value: %d", r);
  544. BUG();
  545. }
  546. }
  547. struct clone_info {
  548. struct mapped_device *md;
  549. struct dm_table *map;
  550. struct bio *bio;
  551. struct dm_io *io;
  552. sector_t sector;
  553. sector_t sector_count;
  554. unsigned short idx;
  555. };
  556. static void dm_bio_destructor(struct bio *bio)
  557. {
  558. struct bio_set *bs = bio->bi_private;
  559. bio_free(bio, bs);
  560. }
  561. /*
  562. * Creates a little bio that is just does part of a bvec.
  563. */
  564. static struct bio *split_bvec(struct bio *bio, sector_t sector,
  565. unsigned short idx, unsigned int offset,
  566. unsigned int len, struct bio_set *bs)
  567. {
  568. struct bio *clone;
  569. struct bio_vec *bv = bio->bi_io_vec + idx;
  570. clone = bio_alloc_bioset(GFP_NOIO, 1, bs);
  571. clone->bi_destructor = dm_bio_destructor;
  572. *clone->bi_io_vec = *bv;
  573. clone->bi_sector = sector;
  574. clone->bi_bdev = bio->bi_bdev;
  575. clone->bi_rw = bio->bi_rw;
  576. clone->bi_vcnt = 1;
  577. clone->bi_size = to_bytes(len);
  578. clone->bi_io_vec->bv_offset = offset;
  579. clone->bi_io_vec->bv_len = clone->bi_size;
  580. clone->bi_flags |= 1 << BIO_CLONED;
  581. return clone;
  582. }
  583. /*
  584. * Creates a bio that consists of range of complete bvecs.
  585. */
  586. static struct bio *clone_bio(struct bio *bio, sector_t sector,
  587. unsigned short idx, unsigned short bv_count,
  588. unsigned int len, struct bio_set *bs)
  589. {
  590. struct bio *clone;
  591. clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs);
  592. __bio_clone(clone, bio);
  593. clone->bi_destructor = dm_bio_destructor;
  594. clone->bi_sector = sector;
  595. clone->bi_idx = idx;
  596. clone->bi_vcnt = idx + bv_count;
  597. clone->bi_size = to_bytes(len);
  598. clone->bi_flags &= ~(1 << BIO_SEG_VALID);
  599. return clone;
  600. }
  601. static int __clone_and_map(struct clone_info *ci)
  602. {
  603. struct bio *clone, *bio = ci->bio;
  604. struct dm_target *ti;
  605. sector_t len = 0, max;
  606. struct dm_target_io *tio;
  607. ti = dm_table_find_target(ci->map, ci->sector);
  608. if (!dm_target_is_valid(ti))
  609. return -EIO;
  610. max = max_io_len(ci->md, ci->sector, ti);
  611. /*
  612. * Allocate a target io object.
  613. */
  614. tio = alloc_tio(ci->md);
  615. tio->io = ci->io;
  616. tio->ti = ti;
  617. memset(&tio->info, 0, sizeof(tio->info));
  618. if (ci->sector_count <= max) {
  619. /*
  620. * Optimise for the simple case where we can do all of
  621. * the remaining io with a single clone.
  622. */
  623. clone = clone_bio(bio, ci->sector, ci->idx,
  624. bio->bi_vcnt - ci->idx, ci->sector_count,
  625. ci->md->bs);
  626. __map_bio(ti, clone, tio);
  627. ci->sector_count = 0;
  628. } else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
  629. /*
  630. * There are some bvecs that don't span targets.
  631. * Do as many of these as possible.
  632. */
  633. int i;
  634. sector_t remaining = max;
  635. sector_t bv_len;
  636. for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) {
  637. bv_len = to_sector(bio->bi_io_vec[i].bv_len);
  638. if (bv_len > remaining)
  639. break;
  640. remaining -= bv_len;
  641. len += bv_len;
  642. }
  643. clone = clone_bio(bio, ci->sector, ci->idx, i - ci->idx, len,
  644. ci->md->bs);
  645. __map_bio(ti, clone, tio);
  646. ci->sector += len;
  647. ci->sector_count -= len;
  648. ci->idx = i;
  649. } else {
  650. /*
  651. * Handle a bvec that must be split between two or more targets.
  652. */
  653. struct bio_vec *bv = bio->bi_io_vec + ci->idx;
  654. sector_t remaining = to_sector(bv->bv_len);
  655. unsigned int offset = 0;
  656. do {
  657. if (offset) {
  658. ti = dm_table_find_target(ci->map, ci->sector);
  659. if (!dm_target_is_valid(ti))
  660. return -EIO;
  661. max = max_io_len(ci->md, ci->sector, ti);
  662. tio = alloc_tio(ci->md);
  663. tio->io = ci->io;
  664. tio->ti = ti;
  665. memset(&tio->info, 0, sizeof(tio->info));
  666. }
  667. len = min(remaining, max);
  668. clone = split_bvec(bio, ci->sector, ci->idx,
  669. bv->bv_offset + offset, len,
  670. ci->md->bs);
  671. __map_bio(ti, clone, tio);
  672. ci->sector += len;
  673. ci->sector_count -= len;
  674. offset += to_bytes(len);
  675. } while (remaining -= len);
  676. ci->idx++;
  677. }
  678. return 0;
  679. }
  680. /*
  681. * Split the bio into several clones.
  682. */
  683. static int __split_bio(struct mapped_device *md, struct bio *bio)
  684. {
  685. struct clone_info ci;
  686. int error = 0;
  687. ci.map = dm_get_table(md);
  688. if (unlikely(!ci.map))
  689. return -EIO;
  690. if (unlikely(bio_barrier(bio) && !dm_table_barrier_ok(ci.map))) {
  691. dm_table_put(ci.map);
  692. bio_endio(bio, -EOPNOTSUPP);
  693. return 0;
  694. }
  695. ci.md = md;
  696. ci.bio = bio;
  697. ci.io = alloc_io(md);
  698. ci.io->error = 0;
  699. atomic_set(&ci.io->io_count, 1);
  700. ci.io->bio = bio;
  701. ci.io->md = md;
  702. ci.sector = bio->bi_sector;
  703. ci.sector_count = bio_sectors(bio);
  704. ci.idx = bio->bi_idx;
  705. start_io_acct(ci.io);
  706. while (ci.sector_count && !error)
  707. error = __clone_and_map(&ci);
  708. /* drop the extra reference count */
  709. dec_pending(ci.io, error);
  710. dm_table_put(ci.map);
  711. return 0;
  712. }
  713. /*-----------------------------------------------------------------
  714. * CRUD END
  715. *---------------------------------------------------------------*/
  716. static int dm_merge_bvec(struct request_queue *q,
  717. struct bvec_merge_data *bvm,
  718. struct bio_vec *biovec)
  719. {
  720. struct mapped_device *md = q->queuedata;
  721. struct dm_table *map = dm_get_table(md);
  722. struct dm_target *ti;
  723. sector_t max_sectors;
  724. int max_size = 0;
  725. if (unlikely(!map))
  726. goto out;
  727. ti = dm_table_find_target(map, bvm->bi_sector);
  728. if (!dm_target_is_valid(ti))
  729. goto out_table;
  730. /*
  731. * Find maximum amount of I/O that won't need splitting
  732. */
  733. max_sectors = min(max_io_len(md, bvm->bi_sector, ti),
  734. (sector_t) BIO_MAX_SECTORS);
  735. max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size;
  736. if (max_size < 0)
  737. max_size = 0;
  738. /*
  739. * merge_bvec_fn() returns number of bytes
  740. * it can accept at this offset
  741. * max is precomputed maximal io size
  742. */
  743. if (max_size && ti->type->merge)
  744. max_size = ti->type->merge(ti, bvm, biovec, max_size);
  745. out_table:
  746. dm_table_put(map);
  747. out:
  748. /*
  749. * Always allow an entire first page
  750. */
  751. if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT))
  752. max_size = biovec->bv_len;
  753. return max_size;
  754. }
  755. /*
  756. * The request function that just remaps the bio built up by
  757. * dm_merge_bvec.
  758. */
  759. static int dm_request(struct request_queue *q, struct bio *bio)
  760. {
  761. int r = -EIO;
  762. int rw = bio_data_dir(bio);
  763. struct mapped_device *md = q->queuedata;
  764. int cpu;
  765. down_read(&md->io_lock);
  766. cpu = part_stat_lock();
  767. part_stat_inc(cpu, &dm_disk(md)->part0, ios[rw]);
  768. part_stat_add(cpu, &dm_disk(md)->part0, sectors[rw], bio_sectors(bio));
  769. part_stat_unlock();
  770. /*
  771. * If we're suspended we have to queue
  772. * this io for later.
  773. */
  774. while (test_bit(DMF_BLOCK_IO, &md->flags)) {
  775. up_read(&md->io_lock);
  776. if (bio_rw(bio) != READA)
  777. r = queue_io(md, bio);
  778. if (r <= 0)
  779. goto out_req;
  780. /*
  781. * We're in a while loop, because someone could suspend
  782. * before we get to the following read lock.
  783. */
  784. down_read(&md->io_lock);
  785. }
  786. r = __split_bio(md, bio);
  787. up_read(&md->io_lock);
  788. out_req:
  789. if (r < 0)
  790. bio_io_error(bio);
  791. return 0;
  792. }
  793. static void dm_unplug_all(struct request_queue *q)
  794. {
  795. struct mapped_device *md = q->queuedata;
  796. struct dm_table *map = dm_get_table(md);
  797. if (map) {
  798. dm_table_unplug_all(map);
  799. dm_table_put(map);
  800. }
  801. }
  802. static int dm_any_congested(void *congested_data, int bdi_bits)
  803. {
  804. int r = bdi_bits;
  805. struct mapped_device *md = congested_data;
  806. struct dm_table *map;
  807. if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
  808. map = dm_get_table(md);
  809. if (map) {
  810. r = dm_table_any_congested(map, bdi_bits);
  811. dm_table_put(map);
  812. }
  813. }
  814. return r;
  815. }
  816. /*-----------------------------------------------------------------
  817. * An IDR is used to keep track of allocated minor numbers.
  818. *---------------------------------------------------------------*/
  819. static DEFINE_IDR(_minor_idr);
  820. static void free_minor(int minor)
  821. {
  822. spin_lock(&_minor_lock);
  823. idr_remove(&_minor_idr, minor);
  824. spin_unlock(&_minor_lock);
  825. }
  826. /*
  827. * See if the device with a specific minor # is free.
  828. */
  829. static int specific_minor(int minor)
  830. {
  831. int r, m;
  832. if (minor >= (1 << MINORBITS))
  833. return -EINVAL;
  834. r = idr_pre_get(&_minor_idr, GFP_KERNEL);
  835. if (!r)
  836. return -ENOMEM;
  837. spin_lock(&_minor_lock);
  838. if (idr_find(&_minor_idr, minor)) {
  839. r = -EBUSY;
  840. goto out;
  841. }
  842. r = idr_get_new_above(&_minor_idr, MINOR_ALLOCED, minor, &m);
  843. if (r)
  844. goto out;
  845. if (m != minor) {
  846. idr_remove(&_minor_idr, m);
  847. r = -EBUSY;
  848. goto out;
  849. }
  850. out:
  851. spin_unlock(&_minor_lock);
  852. return r;
  853. }
  854. static int next_free_minor(int *minor)
  855. {
  856. int r, m;
  857. r = idr_pre_get(&_minor_idr, GFP_KERNEL);
  858. if (!r)
  859. return -ENOMEM;
  860. spin_lock(&_minor_lock);
  861. r = idr_get_new(&_minor_idr, MINOR_ALLOCED, &m);
  862. if (r)
  863. goto out;
  864. if (m >= (1 << MINORBITS)) {
  865. idr_remove(&_minor_idr, m);
  866. r = -ENOSPC;
  867. goto out;
  868. }
  869. *minor = m;
  870. out:
  871. spin_unlock(&_minor_lock);
  872. return r;
  873. }
  874. static struct block_device_operations dm_blk_dops;
  875. /*
  876. * Allocate and initialise a blank device with a given minor.
  877. */
  878. static struct mapped_device *alloc_dev(int minor)
  879. {
  880. int r;
  881. struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL);
  882. void *old_md;
  883. if (!md) {
  884. DMWARN("unable to allocate device, out of memory.");
  885. return NULL;
  886. }
  887. if (!try_module_get(THIS_MODULE))
  888. goto bad_module_get;
  889. /* get a minor number for the dev */
  890. if (minor == DM_ANY_MINOR)
  891. r = next_free_minor(&minor);
  892. else
  893. r = specific_minor(minor);
  894. if (r < 0)
  895. goto bad_minor;
  896. init_rwsem(&md->io_lock);
  897. mutex_init(&md->suspend_lock);
  898. spin_lock_init(&md->pushback_lock);
  899. rwlock_init(&md->map_lock);
  900. atomic_set(&md->holders, 1);
  901. atomic_set(&md->open_count, 0);
  902. atomic_set(&md->event_nr, 0);
  903. atomic_set(&md->uevent_seq, 0);
  904. INIT_LIST_HEAD(&md->uevent_list);
  905. spin_lock_init(&md->uevent_lock);
  906. md->queue = blk_alloc_queue(GFP_KERNEL);
  907. if (!md->queue)
  908. goto bad_queue;
  909. md->queue->queuedata = md;
  910. md->queue->backing_dev_info.congested_fn = dm_any_congested;
  911. md->queue->backing_dev_info.congested_data = md;
  912. blk_queue_make_request(md->queue, dm_request);
  913. blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
  914. md->queue->unplug_fn = dm_unplug_all;
  915. blk_queue_merge_bvec(md->queue, dm_merge_bvec);
  916. md->io_pool = mempool_create_slab_pool(MIN_IOS, _io_cache);
  917. if (!md->io_pool)
  918. goto bad_io_pool;
  919. md->tio_pool = mempool_create_slab_pool(MIN_IOS, _tio_cache);
  920. if (!md->tio_pool)
  921. goto bad_tio_pool;
  922. md->bs = bioset_create(16, 0);
  923. if (!md->bs)
  924. goto bad_no_bioset;
  925. md->disk = alloc_disk(1);
  926. if (!md->disk)
  927. goto bad_disk;
  928. atomic_set(&md->pending, 0);
  929. init_waitqueue_head(&md->wait);
  930. init_waitqueue_head(&md->eventq);
  931. md->disk->major = _major;
  932. md->disk->first_minor = minor;
  933. md->disk->fops = &dm_blk_dops;
  934. md->disk->queue = md->queue;
  935. md->disk->private_data = md;
  936. sprintf(md->disk->disk_name, "dm-%d", minor);
  937. add_disk(md->disk);
  938. format_dev_t(md->name, MKDEV(_major, minor));
  939. md->wq = create_singlethread_workqueue("kdmflush");
  940. if (!md->wq)
  941. goto bad_thread;
  942. /* Populate the mapping, nobody knows we exist yet */
  943. spin_lock(&_minor_lock);
  944. old_md = idr_replace(&_minor_idr, md, minor);
  945. spin_unlock(&_minor_lock);
  946. BUG_ON(old_md != MINOR_ALLOCED);
  947. return md;
  948. bad_thread:
  949. put_disk(md->disk);
  950. bad_disk:
  951. bioset_free(md->bs);
  952. bad_no_bioset:
  953. mempool_destroy(md->tio_pool);
  954. bad_tio_pool:
  955. mempool_destroy(md->io_pool);
  956. bad_io_pool:
  957. blk_cleanup_queue(md->queue);
  958. bad_queue:
  959. free_minor(minor);
  960. bad_minor:
  961. module_put(THIS_MODULE);
  962. bad_module_get:
  963. kfree(md);
  964. return NULL;
  965. }
  966. static void unlock_fs(struct mapped_device *md);
  967. static void free_dev(struct mapped_device *md)
  968. {
  969. int minor = MINOR(disk_devt(md->disk));
  970. if (md->suspended_bdev) {
  971. unlock_fs(md);
  972. bdput(md->suspended_bdev);
  973. }
  974. destroy_workqueue(md->wq);
  975. mempool_destroy(md->tio_pool);
  976. mempool_destroy(md->io_pool);
  977. bioset_free(md->bs);
  978. del_gendisk(md->disk);
  979. free_minor(minor);
  980. spin_lock(&_minor_lock);
  981. md->disk->private_data = NULL;
  982. spin_unlock(&_minor_lock);
  983. put_disk(md->disk);
  984. blk_cleanup_queue(md->queue);
  985. module_put(THIS_MODULE);
  986. kfree(md);
  987. }
  988. /*
  989. * Bind a table to the device.
  990. */
  991. static void event_callback(void *context)
  992. {
  993. unsigned long flags;
  994. LIST_HEAD(uevents);
  995. struct mapped_device *md = (struct mapped_device *) context;
  996. spin_lock_irqsave(&md->uevent_lock, flags);
  997. list_splice_init(&md->uevent_list, &uevents);
  998. spin_unlock_irqrestore(&md->uevent_lock, flags);
  999. dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
  1000. atomic_inc(&md->event_nr);
  1001. wake_up(&md->eventq);
  1002. }
  1003. static void __set_size(struct mapped_device *md, sector_t size)
  1004. {
  1005. set_capacity(md->disk, size);
  1006. mutex_lock(&md->suspended_bdev->bd_inode->i_mutex);
  1007. i_size_write(md->suspended_bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
  1008. mutex_unlock(&md->suspended_bdev->bd_inode->i_mutex);
  1009. }
  1010. static int __bind(struct mapped_device *md, struct dm_table *t)
  1011. {
  1012. struct request_queue *q = md->queue;
  1013. sector_t size;
  1014. size = dm_table_get_size(t);
  1015. /*
  1016. * Wipe any geometry if the size of the table changed.
  1017. */
  1018. if (size != get_capacity(md->disk))
  1019. memset(&md->geometry, 0, sizeof(md->geometry));
  1020. if (md->suspended_bdev)
  1021. __set_size(md, size);
  1022. if (!size) {
  1023. dm_table_destroy(t);
  1024. return 0;
  1025. }
  1026. dm_table_event_callback(t, event_callback, md);
  1027. write_lock(&md->map_lock);
  1028. md->map = t;
  1029. dm_table_set_restrictions(t, q);
  1030. write_unlock(&md->map_lock);
  1031. return 0;
  1032. }
  1033. static void __unbind(struct mapped_device *md)
  1034. {
  1035. struct dm_table *map = md->map;
  1036. if (!map)
  1037. return;
  1038. dm_table_event_callback(map, NULL, NULL);
  1039. write_lock(&md->map_lock);
  1040. md->map = NULL;
  1041. write_unlock(&md->map_lock);
  1042. dm_table_destroy(map);
  1043. }
  1044. /*
  1045. * Constructor for a new device.
  1046. */
  1047. int dm_create(int minor, struct mapped_device **result)
  1048. {
  1049. struct mapped_device *md;
  1050. md = alloc_dev(minor);
  1051. if (!md)
  1052. return -ENXIO;
  1053. dm_sysfs_init(md);
  1054. *result = md;
  1055. return 0;
  1056. }
  1057. static struct mapped_device *dm_find_md(dev_t dev)
  1058. {
  1059. struct mapped_device *md;
  1060. unsigned minor = MINOR(dev);
  1061. if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
  1062. return NULL;
  1063. spin_lock(&_minor_lock);
  1064. md = idr_find(&_minor_idr, minor);
  1065. if (md && (md == MINOR_ALLOCED ||
  1066. (MINOR(disk_devt(dm_disk(md))) != minor) ||
  1067. test_bit(DMF_FREEING, &md->flags))) {
  1068. md = NULL;
  1069. goto out;
  1070. }
  1071. out:
  1072. spin_unlock(&_minor_lock);
  1073. return md;
  1074. }
  1075. struct mapped_device *dm_get_md(dev_t dev)
  1076. {
  1077. struct mapped_device *md = dm_find_md(dev);
  1078. if (md)
  1079. dm_get(md);
  1080. return md;
  1081. }
  1082. void *dm_get_mdptr(struct mapped_device *md)
  1083. {
  1084. return md->interface_ptr;
  1085. }
  1086. void dm_set_mdptr(struct mapped_device *md, void *ptr)
  1087. {
  1088. md->interface_ptr = ptr;
  1089. }
  1090. void dm_get(struct mapped_device *md)
  1091. {
  1092. atomic_inc(&md->holders);
  1093. }
  1094. const char *dm_device_name(struct mapped_device *md)
  1095. {
  1096. return md->name;
  1097. }
  1098. EXPORT_SYMBOL_GPL(dm_device_name);
  1099. void dm_put(struct mapped_device *md)
  1100. {
  1101. struct dm_table *map;
  1102. BUG_ON(test_bit(DMF_FREEING, &md->flags));
  1103. if (atomic_dec_and_lock(&md->holders, &_minor_lock)) {
  1104. map = dm_get_table(md);
  1105. idr_replace(&_minor_idr, MINOR_ALLOCED,
  1106. MINOR(disk_devt(dm_disk(md))));
  1107. set_bit(DMF_FREEING, &md->flags);
  1108. spin_unlock(&_minor_lock);
  1109. if (!dm_suspended(md)) {
  1110. dm_table_presuspend_targets(map);
  1111. dm_table_postsuspend_targets(map);
  1112. }
  1113. dm_sysfs_exit(md);
  1114. dm_table_put(map);
  1115. __unbind(md);
  1116. free_dev(md);
  1117. }
  1118. }
  1119. EXPORT_SYMBOL_GPL(dm_put);
  1120. static int dm_wait_for_completion(struct mapped_device *md)
  1121. {
  1122. int r = 0;
  1123. while (1) {
  1124. set_current_state(TASK_INTERRUPTIBLE);
  1125. smp_mb();
  1126. if (!atomic_read(&md->pending))
  1127. break;
  1128. if (signal_pending(current)) {
  1129. r = -EINTR;
  1130. break;
  1131. }
  1132. io_schedule();
  1133. }
  1134. set_current_state(TASK_RUNNING);
  1135. return r;
  1136. }
  1137. /*
  1138. * Process the deferred bios
  1139. */
  1140. static void __flush_deferred_io(struct mapped_device *md)
  1141. {
  1142. struct bio *c;
  1143. while ((c = bio_list_pop(&md->deferred))) {
  1144. if (__split_bio(md, c))
  1145. bio_io_error(c);
  1146. }
  1147. clear_bit(DMF_BLOCK_IO, &md->flags);
  1148. }
  1149. static void __merge_pushback_list(struct mapped_device *md)
  1150. {
  1151. unsigned long flags;
  1152. spin_lock_irqsave(&md->pushback_lock, flags);
  1153. clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
  1154. bio_list_merge_head(&md->deferred, &md->pushback);
  1155. bio_list_init(&md->pushback);
  1156. spin_unlock_irqrestore(&md->pushback_lock, flags);
  1157. }
  1158. static void dm_wq_work(struct work_struct *work)
  1159. {
  1160. struct dm_wq_req *req = container_of(work, struct dm_wq_req, work);
  1161. struct mapped_device *md = req->md;
  1162. down_write(&md->io_lock);
  1163. switch (req->type) {
  1164. case DM_WQ_FLUSH_DEFERRED:
  1165. __flush_deferred_io(md);
  1166. break;
  1167. default:
  1168. DMERR("dm_wq_work: unrecognised work type %d", req->type);
  1169. BUG();
  1170. }
  1171. up_write(&md->io_lock);
  1172. }
  1173. static void dm_wq_queue(struct mapped_device *md, int type, void *context,
  1174. struct dm_wq_req *req)
  1175. {
  1176. req->type = type;
  1177. req->md = md;
  1178. req->context = context;
  1179. INIT_WORK(&req->work, dm_wq_work);
  1180. queue_work(md->wq, &req->work);
  1181. }
  1182. static void dm_queue_flush(struct mapped_device *md, int type, void *context)
  1183. {
  1184. struct dm_wq_req req;
  1185. dm_wq_queue(md, type, context, &req);
  1186. flush_workqueue(md->wq);
  1187. }
  1188. /*
  1189. * Swap in a new table (destroying old one).
  1190. */
  1191. int dm_swap_table(struct mapped_device *md, struct dm_table *table)
  1192. {
  1193. int r = -EINVAL;
  1194. mutex_lock(&md->suspend_lock);
  1195. /* device must be suspended */
  1196. if (!dm_suspended(md))
  1197. goto out;
  1198. /* without bdev, the device size cannot be changed */
  1199. if (!md->suspended_bdev)
  1200. if (get_capacity(md->disk) != dm_table_get_size(table))
  1201. goto out;
  1202. __unbind(md);
  1203. r = __bind(md, table);
  1204. out:
  1205. mutex_unlock(&md->suspend_lock);
  1206. return r;
  1207. }
  1208. /*
  1209. * Functions to lock and unlock any filesystem running on the
  1210. * device.
  1211. */
  1212. static int lock_fs(struct mapped_device *md)
  1213. {
  1214. int r;
  1215. WARN_ON(md->frozen_sb);
  1216. md->frozen_sb = freeze_bdev(md->suspended_bdev);
  1217. if (IS_ERR(md->frozen_sb)) {
  1218. r = PTR_ERR(md->frozen_sb);
  1219. md->frozen_sb = NULL;
  1220. return r;
  1221. }
  1222. set_bit(DMF_FROZEN, &md->flags);
  1223. /* don't bdput right now, we don't want the bdev
  1224. * to go away while it is locked.
  1225. */
  1226. return 0;
  1227. }
  1228. static void unlock_fs(struct mapped_device *md)
  1229. {
  1230. if (!test_bit(DMF_FROZEN, &md->flags))
  1231. return;
  1232. thaw_bdev(md->suspended_bdev, md->frozen_sb);
  1233. md->frozen_sb = NULL;
  1234. clear_bit(DMF_FROZEN, &md->flags);
  1235. }
  1236. /*
  1237. * We need to be able to change a mapping table under a mounted
  1238. * filesystem. For example we might want to move some data in
  1239. * the background. Before the table can be swapped with
  1240. * dm_bind_table, dm_suspend must be called to flush any in
  1241. * flight bios and ensure that any further io gets deferred.
  1242. */
  1243. int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
  1244. {
  1245. struct dm_table *map = NULL;
  1246. DECLARE_WAITQUEUE(wait, current);
  1247. int r = 0;
  1248. int do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG ? 1 : 0;
  1249. int noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG ? 1 : 0;
  1250. mutex_lock(&md->suspend_lock);
  1251. if (dm_suspended(md)) {
  1252. r = -EINVAL;
  1253. goto out_unlock;
  1254. }
  1255. map = dm_get_table(md);
  1256. /*
  1257. * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
  1258. * This flag is cleared before dm_suspend returns.
  1259. */
  1260. if (noflush)
  1261. set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
  1262. /* This does not get reverted if there's an error later. */
  1263. dm_table_presuspend_targets(map);
  1264. /* bdget() can stall if the pending I/Os are not flushed */
  1265. if (!noflush) {
  1266. md->suspended_bdev = bdget_disk(md->disk, 0);
  1267. if (!md->suspended_bdev) {
  1268. DMWARN("bdget failed in dm_suspend");
  1269. r = -ENOMEM;
  1270. goto out;
  1271. }
  1272. /*
  1273. * Flush I/O to the device. noflush supersedes do_lockfs,
  1274. * because lock_fs() needs to flush I/Os.
  1275. */
  1276. if (do_lockfs) {
  1277. r = lock_fs(md);
  1278. if (r)
  1279. goto out;
  1280. }
  1281. }
  1282. /*
  1283. * First we set the BLOCK_IO flag so no more ios will be mapped.
  1284. */
  1285. down_write(&md->io_lock);
  1286. set_bit(DMF_BLOCK_IO, &md->flags);
  1287. add_wait_queue(&md->wait, &wait);
  1288. up_write(&md->io_lock);
  1289. /* unplug */
  1290. if (map)
  1291. dm_table_unplug_all(map);
  1292. /*
  1293. * Wait for the already-mapped ios to complete.
  1294. */
  1295. r = dm_wait_for_completion(md);
  1296. down_write(&md->io_lock);
  1297. remove_wait_queue(&md->wait, &wait);
  1298. if (noflush)
  1299. __merge_pushback_list(md);
  1300. up_write(&md->io_lock);
  1301. /* were we interrupted ? */
  1302. if (r < 0) {
  1303. dm_queue_flush(md, DM_WQ_FLUSH_DEFERRED, NULL);
  1304. unlock_fs(md);
  1305. goto out; /* pushback list is already flushed, so skip flush */
  1306. }
  1307. dm_table_postsuspend_targets(map);
  1308. set_bit(DMF_SUSPENDED, &md->flags);
  1309. out:
  1310. if (r && md->suspended_bdev) {
  1311. bdput(md->suspended_bdev);
  1312. md->suspended_bdev = NULL;
  1313. }
  1314. dm_table_put(map);
  1315. out_unlock:
  1316. mutex_unlock(&md->suspend_lock);
  1317. return r;
  1318. }
  1319. int dm_resume(struct mapped_device *md)
  1320. {
  1321. int r = -EINVAL;
  1322. struct dm_table *map = NULL;
  1323. mutex_lock(&md->suspend_lock);
  1324. if (!dm_suspended(md))
  1325. goto out;
  1326. map = dm_get_table(md);
  1327. if (!map || !dm_table_get_size(map))
  1328. goto out;
  1329. r = dm_table_resume_targets(map);
  1330. if (r)
  1331. goto out;
  1332. dm_queue_flush(md, DM_WQ_FLUSH_DEFERRED, NULL);
  1333. unlock_fs(md);
  1334. if (md->suspended_bdev) {
  1335. bdput(md->suspended_bdev);
  1336. md->suspended_bdev = NULL;
  1337. }
  1338. clear_bit(DMF_SUSPENDED, &md->flags);
  1339. dm_table_unplug_all(map);
  1340. dm_kobject_uevent(md);
  1341. r = 0;
  1342. out:
  1343. dm_table_put(map);
  1344. mutex_unlock(&md->suspend_lock);
  1345. return r;
  1346. }
  1347. /*-----------------------------------------------------------------
  1348. * Event notification.
  1349. *---------------------------------------------------------------*/
  1350. void dm_kobject_uevent(struct mapped_device *md)
  1351. {
  1352. kobject_uevent(&disk_to_dev(md->disk)->kobj, KOBJ_CHANGE);
  1353. }
  1354. uint32_t dm_next_uevent_seq(struct mapped_device *md)
  1355. {
  1356. return atomic_add_return(1, &md->uevent_seq);
  1357. }
  1358. uint32_t dm_get_event_nr(struct mapped_device *md)
  1359. {
  1360. return atomic_read(&md->event_nr);
  1361. }
  1362. int dm_wait_event(struct mapped_device *md, int event_nr)
  1363. {
  1364. return wait_event_interruptible(md->eventq,
  1365. (event_nr != atomic_read(&md->event_nr)));
  1366. }
  1367. void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
  1368. {
  1369. unsigned long flags;
  1370. spin_lock_irqsave(&md->uevent_lock, flags);
  1371. list_add(elist, &md->uevent_list);
  1372. spin_unlock_irqrestore(&md->uevent_lock, flags);
  1373. }
  1374. /*
  1375. * The gendisk is only valid as long as you have a reference
  1376. * count on 'md'.
  1377. */
  1378. struct gendisk *dm_disk(struct mapped_device *md)
  1379. {
  1380. return md->disk;
  1381. }
  1382. struct kobject *dm_kobject(struct mapped_device *md)
  1383. {
  1384. return &md->kobj;
  1385. }
  1386. /*
  1387. * struct mapped_device should not be exported outside of dm.c
  1388. * so use this check to verify that kobj is part of md structure
  1389. */
  1390. struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
  1391. {
  1392. struct mapped_device *md;
  1393. md = container_of(kobj, struct mapped_device, kobj);
  1394. if (&md->kobj != kobj)
  1395. return NULL;
  1396. dm_get(md);
  1397. return md;
  1398. }
  1399. int dm_suspended(struct mapped_device *md)
  1400. {
  1401. return test_bit(DMF_SUSPENDED, &md->flags);
  1402. }
  1403. int dm_noflush_suspending(struct dm_target *ti)
  1404. {
  1405. struct mapped_device *md = dm_table_get_md(ti->table);
  1406. int r = __noflush_suspending(md);
  1407. dm_put(md);
  1408. return r;
  1409. }
  1410. EXPORT_SYMBOL_GPL(dm_noflush_suspending);
  1411. static struct block_device_operations dm_blk_dops = {
  1412. .open = dm_blk_open,
  1413. .release = dm_blk_close,
  1414. .ioctl = dm_blk_ioctl,
  1415. .getgeo = dm_blk_getgeo,
  1416. .owner = THIS_MODULE
  1417. };
  1418. EXPORT_SYMBOL(dm_get_mapinfo);
  1419. /*
  1420. * module hooks
  1421. */
  1422. module_init(dm_init);
  1423. module_exit(dm_exit);
  1424. module_param(major, uint, 0);
  1425. MODULE_PARM_DESC(major, "The major number of the device mapper");
  1426. MODULE_DESCRIPTION(DM_NAME " driver");
  1427. MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
  1428. MODULE_LICENSE("GPL");