raid10.c 48 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827
  1. /*
  2. * raid10.c : Multiple Devices driver for Linux
  3. *
  4. * Copyright (C) 2000-2004 Neil Brown
  5. *
  6. * RAID-10 support for md.
  7. *
  8. * Base on code in raid1.c. See raid1.c for futher copyright information.
  9. *
  10. *
  11. * This program is free software; you can redistribute it and/or modify
  12. * it under the terms of the GNU General Public License as published by
  13. * the Free Software Foundation; either version 2, or (at your option)
  14. * any later version.
  15. *
  16. * You should have received a copy of the GNU General Public License
  17. * (for example /usr/src/linux/COPYING); if not, write to the Free
  18. * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19. */
  20. #include <linux/raid/raid10.h>
  21. /*
  22. * RAID10 provides a combination of RAID0 and RAID1 functionality.
  23. * The layout of data is defined by
  24. * chunk_size
  25. * raid_disks
  26. * near_copies (stored in low byte of layout)
  27. * far_copies (stored in second byte of layout)
  28. *
  29. * The data to be stored is divided into chunks using chunksize.
  30. * Each device is divided into far_copies sections.
  31. * In each section, chunks are laid out in a style similar to raid0, but
  32. * near_copies copies of each chunk is stored (each on a different drive).
  33. * The starting device for each section is offset near_copies from the starting
  34. * device of the previous section.
  35. * Thus there are (near_copies*far_copies) of each chunk, and each is on a different
  36. * drive.
  37. * near_copies and far_copies must be at least one, and their product is at most
  38. * raid_disks.
  39. */
  40. /*
  41. * Number of guaranteed r10bios in case of extreme VM load:
  42. */
  43. #define NR_RAID10_BIOS 256
  44. static void unplug_slaves(mddev_t *mddev);
  45. static void * r10bio_pool_alloc(unsigned int __nocast gfp_flags, void *data)
  46. {
  47. conf_t *conf = data;
  48. r10bio_t *r10_bio;
  49. int size = offsetof(struct r10bio_s, devs[conf->copies]);
  50. /* allocate a r10bio with room for raid_disks entries in the bios array */
  51. r10_bio = kmalloc(size, gfp_flags);
  52. if (r10_bio)
  53. memset(r10_bio, 0, size);
  54. else
  55. unplug_slaves(conf->mddev);
  56. return r10_bio;
  57. }
  58. static void r10bio_pool_free(void *r10_bio, void *data)
  59. {
  60. kfree(r10_bio);
  61. }
  62. #define RESYNC_BLOCK_SIZE (64*1024)
  63. //#define RESYNC_BLOCK_SIZE PAGE_SIZE
  64. #define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9)
  65. #define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE)
  66. #define RESYNC_WINDOW (2048*1024)
  67. /*
  68. * When performing a resync, we need to read and compare, so
  69. * we need as many pages are there are copies.
  70. * When performing a recovery, we need 2 bios, one for read,
  71. * one for write (we recover only one drive per r10buf)
  72. *
  73. */
  74. static void * r10buf_pool_alloc(unsigned int __nocast gfp_flags, void *data)
  75. {
  76. conf_t *conf = data;
  77. struct page *page;
  78. r10bio_t *r10_bio;
  79. struct bio *bio;
  80. int i, j;
  81. int nalloc;
  82. r10_bio = r10bio_pool_alloc(gfp_flags, conf);
  83. if (!r10_bio) {
  84. unplug_slaves(conf->mddev);
  85. return NULL;
  86. }
  87. if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery))
  88. nalloc = conf->copies; /* resync */
  89. else
  90. nalloc = 2; /* recovery */
  91. /*
  92. * Allocate bios.
  93. */
  94. for (j = nalloc ; j-- ; ) {
  95. bio = bio_alloc(gfp_flags, RESYNC_PAGES);
  96. if (!bio)
  97. goto out_free_bio;
  98. r10_bio->devs[j].bio = bio;
  99. }
  100. /*
  101. * Allocate RESYNC_PAGES data pages and attach them
  102. * where needed.
  103. */
  104. for (j = 0 ; j < nalloc; j++) {
  105. bio = r10_bio->devs[j].bio;
  106. for (i = 0; i < RESYNC_PAGES; i++) {
  107. page = alloc_page(gfp_flags);
  108. if (unlikely(!page))
  109. goto out_free_pages;
  110. bio->bi_io_vec[i].bv_page = page;
  111. }
  112. }
  113. return r10_bio;
  114. out_free_pages:
  115. for ( ; i > 0 ; i--)
  116. __free_page(bio->bi_io_vec[i-1].bv_page);
  117. while (j--)
  118. for (i = 0; i < RESYNC_PAGES ; i++)
  119. __free_page(r10_bio->devs[j].bio->bi_io_vec[i].bv_page);
  120. j = -1;
  121. out_free_bio:
  122. while ( ++j < nalloc )
  123. bio_put(r10_bio->devs[j].bio);
  124. r10bio_pool_free(r10_bio, conf);
  125. return NULL;
  126. }
  127. static void r10buf_pool_free(void *__r10_bio, void *data)
  128. {
  129. int i;
  130. conf_t *conf = data;
  131. r10bio_t *r10bio = __r10_bio;
  132. int j;
  133. for (j=0; j < conf->copies; j++) {
  134. struct bio *bio = r10bio->devs[j].bio;
  135. if (bio) {
  136. for (i = 0; i < RESYNC_PAGES; i++) {
  137. __free_page(bio->bi_io_vec[i].bv_page);
  138. bio->bi_io_vec[i].bv_page = NULL;
  139. }
  140. bio_put(bio);
  141. }
  142. }
  143. r10bio_pool_free(r10bio, conf);
  144. }
  145. static void put_all_bios(conf_t *conf, r10bio_t *r10_bio)
  146. {
  147. int i;
  148. for (i = 0; i < conf->copies; i++) {
  149. struct bio **bio = & r10_bio->devs[i].bio;
  150. if (*bio)
  151. bio_put(*bio);
  152. *bio = NULL;
  153. }
  154. }
  155. static inline void free_r10bio(r10bio_t *r10_bio)
  156. {
  157. unsigned long flags;
  158. conf_t *conf = mddev_to_conf(r10_bio->mddev);
  159. /*
  160. * Wake up any possible resync thread that waits for the device
  161. * to go idle.
  162. */
  163. spin_lock_irqsave(&conf->resync_lock, flags);
  164. if (!--conf->nr_pending) {
  165. wake_up(&conf->wait_idle);
  166. wake_up(&conf->wait_resume);
  167. }
  168. spin_unlock_irqrestore(&conf->resync_lock, flags);
  169. put_all_bios(conf, r10_bio);
  170. mempool_free(r10_bio, conf->r10bio_pool);
  171. }
  172. static inline void put_buf(r10bio_t *r10_bio)
  173. {
  174. conf_t *conf = mddev_to_conf(r10_bio->mddev);
  175. unsigned long flags;
  176. mempool_free(r10_bio, conf->r10buf_pool);
  177. spin_lock_irqsave(&conf->resync_lock, flags);
  178. if (!conf->barrier)
  179. BUG();
  180. --conf->barrier;
  181. wake_up(&conf->wait_resume);
  182. wake_up(&conf->wait_idle);
  183. if (!--conf->nr_pending) {
  184. wake_up(&conf->wait_idle);
  185. wake_up(&conf->wait_resume);
  186. }
  187. spin_unlock_irqrestore(&conf->resync_lock, flags);
  188. }
  189. static void reschedule_retry(r10bio_t *r10_bio)
  190. {
  191. unsigned long flags;
  192. mddev_t *mddev = r10_bio->mddev;
  193. conf_t *conf = mddev_to_conf(mddev);
  194. spin_lock_irqsave(&conf->device_lock, flags);
  195. list_add(&r10_bio->retry_list, &conf->retry_list);
  196. spin_unlock_irqrestore(&conf->device_lock, flags);
  197. md_wakeup_thread(mddev->thread);
  198. }
  199. /*
  200. * raid_end_bio_io() is called when we have finished servicing a mirrored
  201. * operation and are ready to return a success/failure code to the buffer
  202. * cache layer.
  203. */
  204. static void raid_end_bio_io(r10bio_t *r10_bio)
  205. {
  206. struct bio *bio = r10_bio->master_bio;
  207. bio_endio(bio, bio->bi_size,
  208. test_bit(R10BIO_Uptodate, &r10_bio->state) ? 0 : -EIO);
  209. free_r10bio(r10_bio);
  210. }
  211. /*
  212. * Update disk head position estimator based on IRQ completion info.
  213. */
  214. static inline void update_head_pos(int slot, r10bio_t *r10_bio)
  215. {
  216. conf_t *conf = mddev_to_conf(r10_bio->mddev);
  217. conf->mirrors[r10_bio->devs[slot].devnum].head_position =
  218. r10_bio->devs[slot].addr + (r10_bio->sectors);
  219. }
  220. static int raid10_end_read_request(struct bio *bio, unsigned int bytes_done, int error)
  221. {
  222. int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
  223. r10bio_t * r10_bio = (r10bio_t *)(bio->bi_private);
  224. int slot, dev;
  225. conf_t *conf = mddev_to_conf(r10_bio->mddev);
  226. if (bio->bi_size)
  227. return 1;
  228. slot = r10_bio->read_slot;
  229. dev = r10_bio->devs[slot].devnum;
  230. /*
  231. * this branch is our 'one mirror IO has finished' event handler:
  232. */
  233. if (!uptodate)
  234. md_error(r10_bio->mddev, conf->mirrors[dev].rdev);
  235. else
  236. /*
  237. * Set R10BIO_Uptodate in our master bio, so that
  238. * we will return a good error code to the higher
  239. * levels even if IO on some other mirrored buffer fails.
  240. *
  241. * The 'master' represents the composite IO operation to
  242. * user-side. So if something waits for IO, then it will
  243. * wait for the 'master' bio.
  244. */
  245. set_bit(R10BIO_Uptodate, &r10_bio->state);
  246. update_head_pos(slot, r10_bio);
  247. /*
  248. * we have only one bio on the read side
  249. */
  250. if (uptodate)
  251. raid_end_bio_io(r10_bio);
  252. else {
  253. /*
  254. * oops, read error:
  255. */
  256. char b[BDEVNAME_SIZE];
  257. if (printk_ratelimit())
  258. printk(KERN_ERR "raid10: %s: rescheduling sector %llu\n",
  259. bdevname(conf->mirrors[dev].rdev->bdev,b), (unsigned long long)r10_bio->sector);
  260. reschedule_retry(r10_bio);
  261. }
  262. rdev_dec_pending(conf->mirrors[dev].rdev, conf->mddev);
  263. return 0;
  264. }
  265. static int raid10_end_write_request(struct bio *bio, unsigned int bytes_done, int error)
  266. {
  267. int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
  268. r10bio_t * r10_bio = (r10bio_t *)(bio->bi_private);
  269. int slot, dev;
  270. conf_t *conf = mddev_to_conf(r10_bio->mddev);
  271. if (bio->bi_size)
  272. return 1;
  273. for (slot = 0; slot < conf->copies; slot++)
  274. if (r10_bio->devs[slot].bio == bio)
  275. break;
  276. dev = r10_bio->devs[slot].devnum;
  277. /*
  278. * this branch is our 'one mirror IO has finished' event handler:
  279. */
  280. if (!uptodate)
  281. md_error(r10_bio->mddev, conf->mirrors[dev].rdev);
  282. else
  283. /*
  284. * Set R10BIO_Uptodate in our master bio, so that
  285. * we will return a good error code for to the higher
  286. * levels even if IO on some other mirrored buffer fails.
  287. *
  288. * The 'master' represents the composite IO operation to
  289. * user-side. So if something waits for IO, then it will
  290. * wait for the 'master' bio.
  291. */
  292. set_bit(R10BIO_Uptodate, &r10_bio->state);
  293. update_head_pos(slot, r10_bio);
  294. /*
  295. *
  296. * Let's see if all mirrored write operations have finished
  297. * already.
  298. */
  299. if (atomic_dec_and_test(&r10_bio->remaining)) {
  300. md_write_end(r10_bio->mddev);
  301. raid_end_bio_io(r10_bio);
  302. }
  303. rdev_dec_pending(conf->mirrors[dev].rdev, conf->mddev);
  304. return 0;
  305. }
  306. /*
  307. * RAID10 layout manager
  308. * Aswell as the chunksize and raid_disks count, there are two
  309. * parameters: near_copies and far_copies.
  310. * near_copies * far_copies must be <= raid_disks.
  311. * Normally one of these will be 1.
  312. * If both are 1, we get raid0.
  313. * If near_copies == raid_disks, we get raid1.
  314. *
  315. * Chunks are layed out in raid0 style with near_copies copies of the
  316. * first chunk, followed by near_copies copies of the next chunk and
  317. * so on.
  318. * If far_copies > 1, then after 1/far_copies of the array has been assigned
  319. * as described above, we start again with a device offset of near_copies.
  320. * So we effectively have another copy of the whole array further down all
  321. * the drives, but with blocks on different drives.
  322. * With this layout, and block is never stored twice on the one device.
  323. *
  324. * raid10_find_phys finds the sector offset of a given virtual sector
  325. * on each device that it is on. If a block isn't on a device,
  326. * that entry in the array is set to MaxSector.
  327. *
  328. * raid10_find_virt does the reverse mapping, from a device and a
  329. * sector offset to a virtual address
  330. */
  331. static void raid10_find_phys(conf_t *conf, r10bio_t *r10bio)
  332. {
  333. int n,f;
  334. sector_t sector;
  335. sector_t chunk;
  336. sector_t stripe;
  337. int dev;
  338. int slot = 0;
  339. /* now calculate first sector/dev */
  340. chunk = r10bio->sector >> conf->chunk_shift;
  341. sector = r10bio->sector & conf->chunk_mask;
  342. chunk *= conf->near_copies;
  343. stripe = chunk;
  344. dev = sector_div(stripe, conf->raid_disks);
  345. sector += stripe << conf->chunk_shift;
  346. /* and calculate all the others */
  347. for (n=0; n < conf->near_copies; n++) {
  348. int d = dev;
  349. sector_t s = sector;
  350. r10bio->devs[slot].addr = sector;
  351. r10bio->devs[slot].devnum = d;
  352. slot++;
  353. for (f = 1; f < conf->far_copies; f++) {
  354. d += conf->near_copies;
  355. if (d >= conf->raid_disks)
  356. d -= conf->raid_disks;
  357. s += conf->stride;
  358. r10bio->devs[slot].devnum = d;
  359. r10bio->devs[slot].addr = s;
  360. slot++;
  361. }
  362. dev++;
  363. if (dev >= conf->raid_disks) {
  364. dev = 0;
  365. sector += (conf->chunk_mask + 1);
  366. }
  367. }
  368. BUG_ON(slot != conf->copies);
  369. }
  370. static sector_t raid10_find_virt(conf_t *conf, sector_t sector, int dev)
  371. {
  372. sector_t offset, chunk, vchunk;
  373. while (sector > conf->stride) {
  374. sector -= conf->stride;
  375. if (dev < conf->near_copies)
  376. dev += conf->raid_disks - conf->near_copies;
  377. else
  378. dev -= conf->near_copies;
  379. }
  380. offset = sector & conf->chunk_mask;
  381. chunk = sector >> conf->chunk_shift;
  382. vchunk = chunk * conf->raid_disks + dev;
  383. sector_div(vchunk, conf->near_copies);
  384. return (vchunk << conf->chunk_shift) + offset;
  385. }
  386. /**
  387. * raid10_mergeable_bvec -- tell bio layer if a two requests can be merged
  388. * @q: request queue
  389. * @bio: the buffer head that's been built up so far
  390. * @biovec: the request that could be merged to it.
  391. *
  392. * Return amount of bytes we can accept at this offset
  393. * If near_copies == raid_disk, there are no striping issues,
  394. * but in that case, the function isn't called at all.
  395. */
  396. static int raid10_mergeable_bvec(request_queue_t *q, struct bio *bio,
  397. struct bio_vec *bio_vec)
  398. {
  399. mddev_t *mddev = q->queuedata;
  400. sector_t sector = bio->bi_sector + get_start_sect(bio->bi_bdev);
  401. int max;
  402. unsigned int chunk_sectors = mddev->chunk_size >> 9;
  403. unsigned int bio_sectors = bio->bi_size >> 9;
  404. max = (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9;
  405. if (max < 0) max = 0; /* bio_add cannot handle a negative return */
  406. if (max <= bio_vec->bv_len && bio_sectors == 0)
  407. return bio_vec->bv_len;
  408. else
  409. return max;
  410. }
  411. /*
  412. * This routine returns the disk from which the requested read should
  413. * be done. There is a per-array 'next expected sequential IO' sector
  414. * number - if this matches on the next IO then we use the last disk.
  415. * There is also a per-disk 'last know head position' sector that is
  416. * maintained from IRQ contexts, both the normal and the resync IO
  417. * completion handlers update this position correctly. If there is no
  418. * perfect sequential match then we pick the disk whose head is closest.
  419. *
  420. * If there are 2 mirrors in the same 2 devices, performance degrades
  421. * because position is mirror, not device based.
  422. *
  423. * The rdev for the device selected will have nr_pending incremented.
  424. */
  425. /*
  426. * FIXME: possibly should rethink readbalancing and do it differently
  427. * depending on near_copies / far_copies geometry.
  428. */
  429. static int read_balance(conf_t *conf, r10bio_t *r10_bio)
  430. {
  431. const unsigned long this_sector = r10_bio->sector;
  432. int disk, slot, nslot;
  433. const int sectors = r10_bio->sectors;
  434. sector_t new_distance, current_distance;
  435. raid10_find_phys(conf, r10_bio);
  436. rcu_read_lock();
  437. /*
  438. * Check if we can balance. We can balance on the whole
  439. * device if no resync is going on, or below the resync window.
  440. * We take the first readable disk when above the resync window.
  441. */
  442. if (conf->mddev->recovery_cp < MaxSector
  443. && (this_sector + sectors >= conf->next_resync)) {
  444. /* make sure that disk is operational */
  445. slot = 0;
  446. disk = r10_bio->devs[slot].devnum;
  447. while (!conf->mirrors[disk].rdev ||
  448. !conf->mirrors[disk].rdev->in_sync) {
  449. slot++;
  450. if (slot == conf->copies) {
  451. slot = 0;
  452. disk = -1;
  453. break;
  454. }
  455. disk = r10_bio->devs[slot].devnum;
  456. }
  457. goto rb_out;
  458. }
  459. /* make sure the disk is operational */
  460. slot = 0;
  461. disk = r10_bio->devs[slot].devnum;
  462. while (!conf->mirrors[disk].rdev ||
  463. !conf->mirrors[disk].rdev->in_sync) {
  464. slot ++;
  465. if (slot == conf->copies) {
  466. disk = -1;
  467. goto rb_out;
  468. }
  469. disk = r10_bio->devs[slot].devnum;
  470. }
  471. current_distance = abs(r10_bio->devs[slot].addr -
  472. conf->mirrors[disk].head_position);
  473. /* Find the disk whose head is closest */
  474. for (nslot = slot; nslot < conf->copies; nslot++) {
  475. int ndisk = r10_bio->devs[nslot].devnum;
  476. if (!conf->mirrors[ndisk].rdev ||
  477. !conf->mirrors[ndisk].rdev->in_sync)
  478. continue;
  479. if (!atomic_read(&conf->mirrors[ndisk].rdev->nr_pending)) {
  480. disk = ndisk;
  481. slot = nslot;
  482. break;
  483. }
  484. new_distance = abs(r10_bio->devs[nslot].addr -
  485. conf->mirrors[ndisk].head_position);
  486. if (new_distance < current_distance) {
  487. current_distance = new_distance;
  488. disk = ndisk;
  489. slot = nslot;
  490. }
  491. }
  492. rb_out:
  493. r10_bio->read_slot = slot;
  494. /* conf->next_seq_sect = this_sector + sectors;*/
  495. if (disk >= 0 && conf->mirrors[disk].rdev)
  496. atomic_inc(&conf->mirrors[disk].rdev->nr_pending);
  497. rcu_read_unlock();
  498. return disk;
  499. }
  500. static void unplug_slaves(mddev_t *mddev)
  501. {
  502. conf_t *conf = mddev_to_conf(mddev);
  503. int i;
  504. rcu_read_lock();
  505. for (i=0; i<mddev->raid_disks; i++) {
  506. mdk_rdev_t *rdev = conf->mirrors[i].rdev;
  507. if (rdev && !rdev->faulty && atomic_read(&rdev->nr_pending)) {
  508. request_queue_t *r_queue = bdev_get_queue(rdev->bdev);
  509. atomic_inc(&rdev->nr_pending);
  510. rcu_read_unlock();
  511. if (r_queue->unplug_fn)
  512. r_queue->unplug_fn(r_queue);
  513. rdev_dec_pending(rdev, mddev);
  514. rcu_read_lock();
  515. }
  516. }
  517. rcu_read_unlock();
  518. }
  519. static void raid10_unplug(request_queue_t *q)
  520. {
  521. unplug_slaves(q->queuedata);
  522. }
  523. static int raid10_issue_flush(request_queue_t *q, struct gendisk *disk,
  524. sector_t *error_sector)
  525. {
  526. mddev_t *mddev = q->queuedata;
  527. conf_t *conf = mddev_to_conf(mddev);
  528. int i, ret = 0;
  529. rcu_read_lock();
  530. for (i=0; i<mddev->raid_disks && ret == 0; i++) {
  531. mdk_rdev_t *rdev = conf->mirrors[i].rdev;
  532. if (rdev && !rdev->faulty) {
  533. struct block_device *bdev = rdev->bdev;
  534. request_queue_t *r_queue = bdev_get_queue(bdev);
  535. if (!r_queue->issue_flush_fn)
  536. ret = -EOPNOTSUPP;
  537. else {
  538. atomic_inc(&rdev->nr_pending);
  539. rcu_read_unlock();
  540. ret = r_queue->issue_flush_fn(r_queue, bdev->bd_disk,
  541. error_sector);
  542. rdev_dec_pending(rdev, mddev);
  543. rcu_read_lock();
  544. }
  545. }
  546. }
  547. rcu_read_unlock();
  548. return ret;
  549. }
  550. /*
  551. * Throttle resync depth, so that we can both get proper overlapping of
  552. * requests, but are still able to handle normal requests quickly.
  553. */
  554. #define RESYNC_DEPTH 32
  555. static void device_barrier(conf_t *conf, sector_t sect)
  556. {
  557. spin_lock_irq(&conf->resync_lock);
  558. wait_event_lock_irq(conf->wait_idle, !waitqueue_active(&conf->wait_resume),
  559. conf->resync_lock, unplug_slaves(conf->mddev));
  560. if (!conf->barrier++) {
  561. wait_event_lock_irq(conf->wait_idle, !conf->nr_pending,
  562. conf->resync_lock, unplug_slaves(conf->mddev));
  563. if (conf->nr_pending)
  564. BUG();
  565. }
  566. wait_event_lock_irq(conf->wait_resume, conf->barrier < RESYNC_DEPTH,
  567. conf->resync_lock, unplug_slaves(conf->mddev));
  568. conf->next_resync = sect;
  569. spin_unlock_irq(&conf->resync_lock);
  570. }
  571. static int make_request(request_queue_t *q, struct bio * bio)
  572. {
  573. mddev_t *mddev = q->queuedata;
  574. conf_t *conf = mddev_to_conf(mddev);
  575. mirror_info_t *mirror;
  576. r10bio_t *r10_bio;
  577. struct bio *read_bio;
  578. int i;
  579. int chunk_sects = conf->chunk_mask + 1;
  580. if (unlikely(bio_barrier(bio))) {
  581. bio_endio(bio, bio->bi_size, -EOPNOTSUPP);
  582. return 0;
  583. }
  584. /* If this request crosses a chunk boundary, we need to
  585. * split it. This will only happen for 1 PAGE (or less) requests.
  586. */
  587. if (unlikely( (bio->bi_sector & conf->chunk_mask) + (bio->bi_size >> 9)
  588. > chunk_sects &&
  589. conf->near_copies < conf->raid_disks)) {
  590. struct bio_pair *bp;
  591. /* Sanity check -- queue functions should prevent this happening */
  592. if (bio->bi_vcnt != 1 ||
  593. bio->bi_idx != 0)
  594. goto bad_map;
  595. /* This is a one page bio that upper layers
  596. * refuse to split for us, so we need to split it.
  597. */
  598. bp = bio_split(bio, bio_split_pool,
  599. chunk_sects - (bio->bi_sector & (chunk_sects - 1)) );
  600. if (make_request(q, &bp->bio1))
  601. generic_make_request(&bp->bio1);
  602. if (make_request(q, &bp->bio2))
  603. generic_make_request(&bp->bio2);
  604. bio_pair_release(bp);
  605. return 0;
  606. bad_map:
  607. printk("raid10_make_request bug: can't convert block across chunks"
  608. " or bigger than %dk %llu %d\n", chunk_sects/2,
  609. (unsigned long long)bio->bi_sector, bio->bi_size >> 10);
  610. bio_io_error(bio, bio->bi_size);
  611. return 0;
  612. }
  613. md_write_start(mddev, bio);
  614. /*
  615. * Register the new request and wait if the reconstruction
  616. * thread has put up a bar for new requests.
  617. * Continue immediately if no resync is active currently.
  618. */
  619. spin_lock_irq(&conf->resync_lock);
  620. wait_event_lock_irq(conf->wait_resume, !conf->barrier, conf->resync_lock, );
  621. conf->nr_pending++;
  622. spin_unlock_irq(&conf->resync_lock);
  623. if (bio_data_dir(bio)==WRITE) {
  624. disk_stat_inc(mddev->gendisk, writes);
  625. disk_stat_add(mddev->gendisk, write_sectors, bio_sectors(bio));
  626. } else {
  627. disk_stat_inc(mddev->gendisk, reads);
  628. disk_stat_add(mddev->gendisk, read_sectors, bio_sectors(bio));
  629. }
  630. r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
  631. r10_bio->master_bio = bio;
  632. r10_bio->sectors = bio->bi_size >> 9;
  633. r10_bio->mddev = mddev;
  634. r10_bio->sector = bio->bi_sector;
  635. if (bio_data_dir(bio) == READ) {
  636. /*
  637. * read balancing logic:
  638. */
  639. int disk = read_balance(conf, r10_bio);
  640. int slot = r10_bio->read_slot;
  641. if (disk < 0) {
  642. raid_end_bio_io(r10_bio);
  643. return 0;
  644. }
  645. mirror = conf->mirrors + disk;
  646. read_bio = bio_clone(bio, GFP_NOIO);
  647. r10_bio->devs[slot].bio = read_bio;
  648. read_bio->bi_sector = r10_bio->devs[slot].addr +
  649. mirror->rdev->data_offset;
  650. read_bio->bi_bdev = mirror->rdev->bdev;
  651. read_bio->bi_end_io = raid10_end_read_request;
  652. read_bio->bi_rw = READ;
  653. read_bio->bi_private = r10_bio;
  654. generic_make_request(read_bio);
  655. return 0;
  656. }
  657. /*
  658. * WRITE:
  659. */
  660. /* first select target devices under spinlock and
  661. * inc refcount on their rdev. Record them by setting
  662. * bios[x] to bio
  663. */
  664. raid10_find_phys(conf, r10_bio);
  665. rcu_read_lock();
  666. for (i = 0; i < conf->copies; i++) {
  667. int d = r10_bio->devs[i].devnum;
  668. if (conf->mirrors[d].rdev &&
  669. !conf->mirrors[d].rdev->faulty) {
  670. atomic_inc(&conf->mirrors[d].rdev->nr_pending);
  671. r10_bio->devs[i].bio = bio;
  672. } else
  673. r10_bio->devs[i].bio = NULL;
  674. }
  675. rcu_read_unlock();
  676. atomic_set(&r10_bio->remaining, 1);
  677. for (i = 0; i < conf->copies; i++) {
  678. struct bio *mbio;
  679. int d = r10_bio->devs[i].devnum;
  680. if (!r10_bio->devs[i].bio)
  681. continue;
  682. mbio = bio_clone(bio, GFP_NOIO);
  683. r10_bio->devs[i].bio = mbio;
  684. mbio->bi_sector = r10_bio->devs[i].addr+
  685. conf->mirrors[d].rdev->data_offset;
  686. mbio->bi_bdev = conf->mirrors[d].rdev->bdev;
  687. mbio->bi_end_io = raid10_end_write_request;
  688. mbio->bi_rw = WRITE;
  689. mbio->bi_private = r10_bio;
  690. atomic_inc(&r10_bio->remaining);
  691. generic_make_request(mbio);
  692. }
  693. if (atomic_dec_and_test(&r10_bio->remaining)) {
  694. md_write_end(mddev);
  695. raid_end_bio_io(r10_bio);
  696. }
  697. return 0;
  698. }
  699. static void status(struct seq_file *seq, mddev_t *mddev)
  700. {
  701. conf_t *conf = mddev_to_conf(mddev);
  702. int i;
  703. if (conf->near_copies < conf->raid_disks)
  704. seq_printf(seq, " %dK chunks", mddev->chunk_size/1024);
  705. if (conf->near_copies > 1)
  706. seq_printf(seq, " %d near-copies", conf->near_copies);
  707. if (conf->far_copies > 1)
  708. seq_printf(seq, " %d far-copies", conf->far_copies);
  709. seq_printf(seq, " [%d/%d] [", conf->raid_disks,
  710. conf->working_disks);
  711. for (i = 0; i < conf->raid_disks; i++)
  712. seq_printf(seq, "%s",
  713. conf->mirrors[i].rdev &&
  714. conf->mirrors[i].rdev->in_sync ? "U" : "_");
  715. seq_printf(seq, "]");
  716. }
  717. static void error(mddev_t *mddev, mdk_rdev_t *rdev)
  718. {
  719. char b[BDEVNAME_SIZE];
  720. conf_t *conf = mddev_to_conf(mddev);
  721. /*
  722. * If it is not operational, then we have already marked it as dead
  723. * else if it is the last working disks, ignore the error, let the
  724. * next level up know.
  725. * else mark the drive as failed
  726. */
  727. if (rdev->in_sync
  728. && conf->working_disks == 1)
  729. /*
  730. * Don't fail the drive, just return an IO error.
  731. * The test should really be more sophisticated than
  732. * "working_disks == 1", but it isn't critical, and
  733. * can wait until we do more sophisticated "is the drive
  734. * really dead" tests...
  735. */
  736. return;
  737. if (rdev->in_sync) {
  738. mddev->degraded++;
  739. conf->working_disks--;
  740. /*
  741. * if recovery is running, make sure it aborts.
  742. */
  743. set_bit(MD_RECOVERY_ERR, &mddev->recovery);
  744. }
  745. rdev->in_sync = 0;
  746. rdev->faulty = 1;
  747. mddev->sb_dirty = 1;
  748. printk(KERN_ALERT "raid10: Disk failure on %s, disabling device. \n"
  749. " Operation continuing on %d devices\n",
  750. bdevname(rdev->bdev,b), conf->working_disks);
  751. }
  752. static void print_conf(conf_t *conf)
  753. {
  754. int i;
  755. mirror_info_t *tmp;
  756. printk("RAID10 conf printout:\n");
  757. if (!conf) {
  758. printk("(!conf)\n");
  759. return;
  760. }
  761. printk(" --- wd:%d rd:%d\n", conf->working_disks,
  762. conf->raid_disks);
  763. for (i = 0; i < conf->raid_disks; i++) {
  764. char b[BDEVNAME_SIZE];
  765. tmp = conf->mirrors + i;
  766. if (tmp->rdev)
  767. printk(" disk %d, wo:%d, o:%d, dev:%s\n",
  768. i, !tmp->rdev->in_sync, !tmp->rdev->faulty,
  769. bdevname(tmp->rdev->bdev,b));
  770. }
  771. }
  772. static void close_sync(conf_t *conf)
  773. {
  774. spin_lock_irq(&conf->resync_lock);
  775. wait_event_lock_irq(conf->wait_resume, !conf->barrier,
  776. conf->resync_lock, unplug_slaves(conf->mddev));
  777. spin_unlock_irq(&conf->resync_lock);
  778. if (conf->barrier) BUG();
  779. if (waitqueue_active(&conf->wait_idle)) BUG();
  780. mempool_destroy(conf->r10buf_pool);
  781. conf->r10buf_pool = NULL;
  782. }
  783. /* check if there are enough drives for
  784. * every block to appear on atleast one
  785. */
  786. static int enough(conf_t *conf)
  787. {
  788. int first = 0;
  789. do {
  790. int n = conf->copies;
  791. int cnt = 0;
  792. while (n--) {
  793. if (conf->mirrors[first].rdev)
  794. cnt++;
  795. first = (first+1) % conf->raid_disks;
  796. }
  797. if (cnt == 0)
  798. return 0;
  799. } while (first != 0);
  800. return 1;
  801. }
  802. static int raid10_spare_active(mddev_t *mddev)
  803. {
  804. int i;
  805. conf_t *conf = mddev->private;
  806. mirror_info_t *tmp;
  807. /*
  808. * Find all non-in_sync disks within the RAID10 configuration
  809. * and mark them in_sync
  810. */
  811. for (i = 0; i < conf->raid_disks; i++) {
  812. tmp = conf->mirrors + i;
  813. if (tmp->rdev
  814. && !tmp->rdev->faulty
  815. && !tmp->rdev->in_sync) {
  816. conf->working_disks++;
  817. mddev->degraded--;
  818. tmp->rdev->in_sync = 1;
  819. }
  820. }
  821. print_conf(conf);
  822. return 0;
  823. }
  824. static int raid10_add_disk(mddev_t *mddev, mdk_rdev_t *rdev)
  825. {
  826. conf_t *conf = mddev->private;
  827. int found = 0;
  828. int mirror;
  829. mirror_info_t *p;
  830. if (mddev->recovery_cp < MaxSector)
  831. /* only hot-add to in-sync arrays, as recovery is
  832. * very different from resync
  833. */
  834. return 0;
  835. if (!enough(conf))
  836. return 0;
  837. for (mirror=0; mirror < mddev->raid_disks; mirror++)
  838. if ( !(p=conf->mirrors+mirror)->rdev) {
  839. blk_queue_stack_limits(mddev->queue,
  840. rdev->bdev->bd_disk->queue);
  841. /* as we don't honour merge_bvec_fn, we must never risk
  842. * violating it, so limit ->max_sector to one PAGE, as
  843. * a one page request is never in violation.
  844. */
  845. if (rdev->bdev->bd_disk->queue->merge_bvec_fn &&
  846. mddev->queue->max_sectors > (PAGE_SIZE>>9))
  847. mddev->queue->max_sectors = (PAGE_SIZE>>9);
  848. p->head_position = 0;
  849. rdev->raid_disk = mirror;
  850. found = 1;
  851. p->rdev = rdev;
  852. break;
  853. }
  854. print_conf(conf);
  855. return found;
  856. }
  857. static int raid10_remove_disk(mddev_t *mddev, int number)
  858. {
  859. conf_t *conf = mddev->private;
  860. int err = 0;
  861. mdk_rdev_t *rdev;
  862. mirror_info_t *p = conf->mirrors+ number;
  863. print_conf(conf);
  864. rdev = p->rdev;
  865. if (rdev) {
  866. if (rdev->in_sync ||
  867. atomic_read(&rdev->nr_pending)) {
  868. err = -EBUSY;
  869. goto abort;
  870. }
  871. p->rdev = NULL;
  872. synchronize_rcu();
  873. if (atomic_read(&rdev->nr_pending)) {
  874. /* lost the race, try later */
  875. err = -EBUSY;
  876. p->rdev = rdev;
  877. }
  878. }
  879. abort:
  880. print_conf(conf);
  881. return err;
  882. }
  883. static int end_sync_read(struct bio *bio, unsigned int bytes_done, int error)
  884. {
  885. int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
  886. r10bio_t * r10_bio = (r10bio_t *)(bio->bi_private);
  887. conf_t *conf = mddev_to_conf(r10_bio->mddev);
  888. int i,d;
  889. if (bio->bi_size)
  890. return 1;
  891. for (i=0; i<conf->copies; i++)
  892. if (r10_bio->devs[i].bio == bio)
  893. break;
  894. if (i == conf->copies)
  895. BUG();
  896. update_head_pos(i, r10_bio);
  897. d = r10_bio->devs[i].devnum;
  898. if (!uptodate)
  899. md_error(r10_bio->mddev,
  900. conf->mirrors[d].rdev);
  901. /* for reconstruct, we always reschedule after a read.
  902. * for resync, only after all reads
  903. */
  904. if (test_bit(R10BIO_IsRecover, &r10_bio->state) ||
  905. atomic_dec_and_test(&r10_bio->remaining)) {
  906. /* we have read all the blocks,
  907. * do the comparison in process context in raid10d
  908. */
  909. reschedule_retry(r10_bio);
  910. }
  911. rdev_dec_pending(conf->mirrors[d].rdev, conf->mddev);
  912. return 0;
  913. }
  914. static int end_sync_write(struct bio *bio, unsigned int bytes_done, int error)
  915. {
  916. int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
  917. r10bio_t * r10_bio = (r10bio_t *)(bio->bi_private);
  918. mddev_t *mddev = r10_bio->mddev;
  919. conf_t *conf = mddev_to_conf(mddev);
  920. int i,d;
  921. if (bio->bi_size)
  922. return 1;
  923. for (i = 0; i < conf->copies; i++)
  924. if (r10_bio->devs[i].bio == bio)
  925. break;
  926. d = r10_bio->devs[i].devnum;
  927. if (!uptodate)
  928. md_error(mddev, conf->mirrors[d].rdev);
  929. update_head_pos(i, r10_bio);
  930. while (atomic_dec_and_test(&r10_bio->remaining)) {
  931. if (r10_bio->master_bio == NULL) {
  932. /* the primary of several recovery bios */
  933. md_done_sync(mddev, r10_bio->sectors, 1);
  934. put_buf(r10_bio);
  935. break;
  936. } else {
  937. r10bio_t *r10_bio2 = (r10bio_t *)r10_bio->master_bio;
  938. put_buf(r10_bio);
  939. r10_bio = r10_bio2;
  940. }
  941. }
  942. rdev_dec_pending(conf->mirrors[d].rdev, mddev);
  943. return 0;
  944. }
  945. /*
  946. * Note: sync and recover and handled very differently for raid10
  947. * This code is for resync.
  948. * For resync, we read through virtual addresses and read all blocks.
  949. * If there is any error, we schedule a write. The lowest numbered
  950. * drive is authoritative.
  951. * However requests come for physical address, so we need to map.
  952. * For every physical address there are raid_disks/copies virtual addresses,
  953. * which is always are least one, but is not necessarly an integer.
  954. * This means that a physical address can span multiple chunks, so we may
  955. * have to submit multiple io requests for a single sync request.
  956. */
  957. /*
  958. * We check if all blocks are in-sync and only write to blocks that
  959. * aren't in sync
  960. */
  961. static void sync_request_write(mddev_t *mddev, r10bio_t *r10_bio)
  962. {
  963. conf_t *conf = mddev_to_conf(mddev);
  964. int i, first;
  965. struct bio *tbio, *fbio;
  966. atomic_set(&r10_bio->remaining, 1);
  967. /* find the first device with a block */
  968. for (i=0; i<conf->copies; i++)
  969. if (test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags))
  970. break;
  971. if (i == conf->copies)
  972. goto done;
  973. first = i;
  974. fbio = r10_bio->devs[i].bio;
  975. /* now find blocks with errors */
  976. for (i=first+1 ; i < conf->copies ; i++) {
  977. int vcnt, j, d;
  978. if (!test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags))
  979. continue;
  980. /* We know that the bi_io_vec layout is the same for
  981. * both 'first' and 'i', so we just compare them.
  982. * All vec entries are PAGE_SIZE;
  983. */
  984. tbio = r10_bio->devs[i].bio;
  985. vcnt = r10_bio->sectors >> (PAGE_SHIFT-9);
  986. for (j = 0; j < vcnt; j++)
  987. if (memcmp(page_address(fbio->bi_io_vec[j].bv_page),
  988. page_address(tbio->bi_io_vec[j].bv_page),
  989. PAGE_SIZE))
  990. break;
  991. if (j == vcnt)
  992. continue;
  993. /* Ok, we need to write this bio
  994. * First we need to fixup bv_offset, bv_len and
  995. * bi_vecs, as the read request might have corrupted these
  996. */
  997. tbio->bi_vcnt = vcnt;
  998. tbio->bi_size = r10_bio->sectors << 9;
  999. tbio->bi_idx = 0;
  1000. tbio->bi_phys_segments = 0;
  1001. tbio->bi_hw_segments = 0;
  1002. tbio->bi_hw_front_size = 0;
  1003. tbio->bi_hw_back_size = 0;
  1004. tbio->bi_flags &= ~(BIO_POOL_MASK - 1);
  1005. tbio->bi_flags |= 1 << BIO_UPTODATE;
  1006. tbio->bi_next = NULL;
  1007. tbio->bi_rw = WRITE;
  1008. tbio->bi_private = r10_bio;
  1009. tbio->bi_sector = r10_bio->devs[i].addr;
  1010. for (j=0; j < vcnt ; j++) {
  1011. tbio->bi_io_vec[j].bv_offset = 0;
  1012. tbio->bi_io_vec[j].bv_len = PAGE_SIZE;
  1013. memcpy(page_address(tbio->bi_io_vec[j].bv_page),
  1014. page_address(fbio->bi_io_vec[j].bv_page),
  1015. PAGE_SIZE);
  1016. }
  1017. tbio->bi_end_io = end_sync_write;
  1018. d = r10_bio->devs[i].devnum;
  1019. atomic_inc(&conf->mirrors[d].rdev->nr_pending);
  1020. atomic_inc(&r10_bio->remaining);
  1021. md_sync_acct(conf->mirrors[d].rdev->bdev, tbio->bi_size >> 9);
  1022. tbio->bi_sector += conf->mirrors[d].rdev->data_offset;
  1023. tbio->bi_bdev = conf->mirrors[d].rdev->bdev;
  1024. generic_make_request(tbio);
  1025. }
  1026. done:
  1027. if (atomic_dec_and_test(&r10_bio->remaining)) {
  1028. md_done_sync(mddev, r10_bio->sectors, 1);
  1029. put_buf(r10_bio);
  1030. }
  1031. }
  1032. /*
  1033. * Now for the recovery code.
  1034. * Recovery happens across physical sectors.
  1035. * We recover all non-is_sync drives by finding the virtual address of
  1036. * each, and then choose a working drive that also has that virt address.
  1037. * There is a separate r10_bio for each non-in_sync drive.
  1038. * Only the first two slots are in use. The first for reading,
  1039. * The second for writing.
  1040. *
  1041. */
  1042. static void recovery_request_write(mddev_t *mddev, r10bio_t *r10_bio)
  1043. {
  1044. conf_t *conf = mddev_to_conf(mddev);
  1045. int i, d;
  1046. struct bio *bio, *wbio;
  1047. /* move the pages across to the second bio
  1048. * and submit the write request
  1049. */
  1050. bio = r10_bio->devs[0].bio;
  1051. wbio = r10_bio->devs[1].bio;
  1052. for (i=0; i < wbio->bi_vcnt; i++) {
  1053. struct page *p = bio->bi_io_vec[i].bv_page;
  1054. bio->bi_io_vec[i].bv_page = wbio->bi_io_vec[i].bv_page;
  1055. wbio->bi_io_vec[i].bv_page = p;
  1056. }
  1057. d = r10_bio->devs[1].devnum;
  1058. atomic_inc(&conf->mirrors[d].rdev->nr_pending);
  1059. md_sync_acct(conf->mirrors[d].rdev->bdev, wbio->bi_size >> 9);
  1060. generic_make_request(wbio);
  1061. }
  1062. /*
  1063. * This is a kernel thread which:
  1064. *
  1065. * 1. Retries failed read operations on working mirrors.
  1066. * 2. Updates the raid superblock when problems encounter.
  1067. * 3. Performs writes following reads for array syncronising.
  1068. */
  1069. static void raid10d(mddev_t *mddev)
  1070. {
  1071. r10bio_t *r10_bio;
  1072. struct bio *bio;
  1073. unsigned long flags;
  1074. conf_t *conf = mddev_to_conf(mddev);
  1075. struct list_head *head = &conf->retry_list;
  1076. int unplug=0;
  1077. mdk_rdev_t *rdev;
  1078. md_check_recovery(mddev);
  1079. for (;;) {
  1080. char b[BDEVNAME_SIZE];
  1081. spin_lock_irqsave(&conf->device_lock, flags);
  1082. if (list_empty(head))
  1083. break;
  1084. r10_bio = list_entry(head->prev, r10bio_t, retry_list);
  1085. list_del(head->prev);
  1086. spin_unlock_irqrestore(&conf->device_lock, flags);
  1087. mddev = r10_bio->mddev;
  1088. conf = mddev_to_conf(mddev);
  1089. if (test_bit(R10BIO_IsSync, &r10_bio->state)) {
  1090. sync_request_write(mddev, r10_bio);
  1091. unplug = 1;
  1092. } else if (test_bit(R10BIO_IsRecover, &r10_bio->state)) {
  1093. recovery_request_write(mddev, r10_bio);
  1094. unplug = 1;
  1095. } else {
  1096. int mirror;
  1097. bio = r10_bio->devs[r10_bio->read_slot].bio;
  1098. r10_bio->devs[r10_bio->read_slot].bio = NULL;
  1099. bio_put(bio);
  1100. mirror = read_balance(conf, r10_bio);
  1101. if (mirror == -1) {
  1102. printk(KERN_ALERT "raid10: %s: unrecoverable I/O"
  1103. " read error for block %llu\n",
  1104. bdevname(bio->bi_bdev,b),
  1105. (unsigned long long)r10_bio->sector);
  1106. raid_end_bio_io(r10_bio);
  1107. } else {
  1108. rdev = conf->mirrors[mirror].rdev;
  1109. if (printk_ratelimit())
  1110. printk(KERN_ERR "raid10: %s: redirecting sector %llu to"
  1111. " another mirror\n",
  1112. bdevname(rdev->bdev,b),
  1113. (unsigned long long)r10_bio->sector);
  1114. bio = bio_clone(r10_bio->master_bio, GFP_NOIO);
  1115. r10_bio->devs[r10_bio->read_slot].bio = bio;
  1116. bio->bi_sector = r10_bio->devs[r10_bio->read_slot].addr
  1117. + rdev->data_offset;
  1118. bio->bi_bdev = rdev->bdev;
  1119. bio->bi_rw = READ;
  1120. bio->bi_private = r10_bio;
  1121. bio->bi_end_io = raid10_end_read_request;
  1122. unplug = 1;
  1123. generic_make_request(bio);
  1124. }
  1125. }
  1126. }
  1127. spin_unlock_irqrestore(&conf->device_lock, flags);
  1128. if (unplug)
  1129. unplug_slaves(mddev);
  1130. }
  1131. static int init_resync(conf_t *conf)
  1132. {
  1133. int buffs;
  1134. buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
  1135. if (conf->r10buf_pool)
  1136. BUG();
  1137. conf->r10buf_pool = mempool_create(buffs, r10buf_pool_alloc, r10buf_pool_free, conf);
  1138. if (!conf->r10buf_pool)
  1139. return -ENOMEM;
  1140. conf->next_resync = 0;
  1141. return 0;
  1142. }
  1143. /*
  1144. * perform a "sync" on one "block"
  1145. *
  1146. * We need to make sure that no normal I/O request - particularly write
  1147. * requests - conflict with active sync requests.
  1148. *
  1149. * This is achieved by tracking pending requests and a 'barrier' concept
  1150. * that can be installed to exclude normal IO requests.
  1151. *
  1152. * Resync and recovery are handled very differently.
  1153. * We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery.
  1154. *
  1155. * For resync, we iterate over virtual addresses, read all copies,
  1156. * and update if there are differences. If only one copy is live,
  1157. * skip it.
  1158. * For recovery, we iterate over physical addresses, read a good
  1159. * value for each non-in_sync drive, and over-write.
  1160. *
  1161. * So, for recovery we may have several outstanding complex requests for a
  1162. * given address, one for each out-of-sync device. We model this by allocating
  1163. * a number of r10_bio structures, one for each out-of-sync device.
  1164. * As we setup these structures, we collect all bio's together into a list
  1165. * which we then process collectively to add pages, and then process again
  1166. * to pass to generic_make_request.
  1167. *
  1168. * The r10_bio structures are linked using a borrowed master_bio pointer.
  1169. * This link is counted in ->remaining. When the r10_bio that points to NULL
  1170. * has its remaining count decremented to 0, the whole complex operation
  1171. * is complete.
  1172. *
  1173. */
  1174. static sector_t sync_request(mddev_t *mddev, sector_t sector_nr, int *skipped, int go_faster)
  1175. {
  1176. conf_t *conf = mddev_to_conf(mddev);
  1177. r10bio_t *r10_bio;
  1178. struct bio *biolist = NULL, *bio;
  1179. sector_t max_sector, nr_sectors;
  1180. int disk;
  1181. int i;
  1182. sector_t sectors_skipped = 0;
  1183. int chunks_skipped = 0;
  1184. if (!conf->r10buf_pool)
  1185. if (init_resync(conf))
  1186. return 0;
  1187. skipped:
  1188. max_sector = mddev->size << 1;
  1189. if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
  1190. max_sector = mddev->resync_max_sectors;
  1191. if (sector_nr >= max_sector) {
  1192. close_sync(conf);
  1193. *skipped = 1;
  1194. return sectors_skipped;
  1195. }
  1196. if (chunks_skipped >= conf->raid_disks) {
  1197. /* if there has been nothing to do on any drive,
  1198. * then there is nothing to do at all..
  1199. */
  1200. *skipped = 1;
  1201. return (max_sector - sector_nr) + sectors_skipped;
  1202. }
  1203. /* make sure whole request will fit in a chunk - if chunks
  1204. * are meaningful
  1205. */
  1206. if (conf->near_copies < conf->raid_disks &&
  1207. max_sector > (sector_nr | conf->chunk_mask))
  1208. max_sector = (sector_nr | conf->chunk_mask) + 1;
  1209. /*
  1210. * If there is non-resync activity waiting for us then
  1211. * put in a delay to throttle resync.
  1212. */
  1213. if (!go_faster && waitqueue_active(&conf->wait_resume))
  1214. msleep_interruptible(1000);
  1215. device_barrier(conf, sector_nr + RESYNC_SECTORS);
  1216. /* Again, very different code for resync and recovery.
  1217. * Both must result in an r10bio with a list of bios that
  1218. * have bi_end_io, bi_sector, bi_bdev set,
  1219. * and bi_private set to the r10bio.
  1220. * For recovery, we may actually create several r10bios
  1221. * with 2 bios in each, that correspond to the bios in the main one.
  1222. * In this case, the subordinate r10bios link back through a
  1223. * borrowed master_bio pointer, and the counter in the master
  1224. * includes a ref from each subordinate.
  1225. */
  1226. /* First, we decide what to do and set ->bi_end_io
  1227. * To end_sync_read if we want to read, and
  1228. * end_sync_write if we will want to write.
  1229. */
  1230. if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
  1231. /* recovery... the complicated one */
  1232. int i, j, k;
  1233. r10_bio = NULL;
  1234. for (i=0 ; i<conf->raid_disks; i++)
  1235. if (conf->mirrors[i].rdev &&
  1236. !conf->mirrors[i].rdev->in_sync) {
  1237. /* want to reconstruct this device */
  1238. r10bio_t *rb2 = r10_bio;
  1239. r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
  1240. spin_lock_irq(&conf->resync_lock);
  1241. conf->nr_pending++;
  1242. if (rb2) conf->barrier++;
  1243. spin_unlock_irq(&conf->resync_lock);
  1244. atomic_set(&r10_bio->remaining, 0);
  1245. r10_bio->master_bio = (struct bio*)rb2;
  1246. if (rb2)
  1247. atomic_inc(&rb2->remaining);
  1248. r10_bio->mddev = mddev;
  1249. set_bit(R10BIO_IsRecover, &r10_bio->state);
  1250. r10_bio->sector = raid10_find_virt(conf, sector_nr, i);
  1251. raid10_find_phys(conf, r10_bio);
  1252. for (j=0; j<conf->copies;j++) {
  1253. int d = r10_bio->devs[j].devnum;
  1254. if (conf->mirrors[d].rdev &&
  1255. conf->mirrors[d].rdev->in_sync) {
  1256. /* This is where we read from */
  1257. bio = r10_bio->devs[0].bio;
  1258. bio->bi_next = biolist;
  1259. biolist = bio;
  1260. bio->bi_private = r10_bio;
  1261. bio->bi_end_io = end_sync_read;
  1262. bio->bi_rw = 0;
  1263. bio->bi_sector = r10_bio->devs[j].addr +
  1264. conf->mirrors[d].rdev->data_offset;
  1265. bio->bi_bdev = conf->mirrors[d].rdev->bdev;
  1266. atomic_inc(&conf->mirrors[d].rdev->nr_pending);
  1267. atomic_inc(&r10_bio->remaining);
  1268. /* and we write to 'i' */
  1269. for (k=0; k<conf->copies; k++)
  1270. if (r10_bio->devs[k].devnum == i)
  1271. break;
  1272. bio = r10_bio->devs[1].bio;
  1273. bio->bi_next = biolist;
  1274. biolist = bio;
  1275. bio->bi_private = r10_bio;
  1276. bio->bi_end_io = end_sync_write;
  1277. bio->bi_rw = 1;
  1278. bio->bi_sector = r10_bio->devs[k].addr +
  1279. conf->mirrors[i].rdev->data_offset;
  1280. bio->bi_bdev = conf->mirrors[i].rdev->bdev;
  1281. r10_bio->devs[0].devnum = d;
  1282. r10_bio->devs[1].devnum = i;
  1283. break;
  1284. }
  1285. }
  1286. if (j == conf->copies) {
  1287. /* Cannot recover, so abort the recovery */
  1288. put_buf(r10_bio);
  1289. r10_bio = rb2;
  1290. if (!test_and_set_bit(MD_RECOVERY_ERR, &mddev->recovery))
  1291. printk(KERN_INFO "raid10: %s: insufficient working devices for recovery.\n",
  1292. mdname(mddev));
  1293. break;
  1294. }
  1295. }
  1296. if (biolist == NULL) {
  1297. while (r10_bio) {
  1298. r10bio_t *rb2 = r10_bio;
  1299. r10_bio = (r10bio_t*) rb2->master_bio;
  1300. rb2->master_bio = NULL;
  1301. put_buf(rb2);
  1302. }
  1303. goto giveup;
  1304. }
  1305. } else {
  1306. /* resync. Schedule a read for every block at this virt offset */
  1307. int count = 0;
  1308. r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
  1309. spin_lock_irq(&conf->resync_lock);
  1310. conf->nr_pending++;
  1311. spin_unlock_irq(&conf->resync_lock);
  1312. r10_bio->mddev = mddev;
  1313. atomic_set(&r10_bio->remaining, 0);
  1314. r10_bio->master_bio = NULL;
  1315. r10_bio->sector = sector_nr;
  1316. set_bit(R10BIO_IsSync, &r10_bio->state);
  1317. raid10_find_phys(conf, r10_bio);
  1318. r10_bio->sectors = (sector_nr | conf->chunk_mask) - sector_nr +1;
  1319. for (i=0; i<conf->copies; i++) {
  1320. int d = r10_bio->devs[i].devnum;
  1321. bio = r10_bio->devs[i].bio;
  1322. bio->bi_end_io = NULL;
  1323. if (conf->mirrors[d].rdev == NULL ||
  1324. conf->mirrors[d].rdev->faulty)
  1325. continue;
  1326. atomic_inc(&conf->mirrors[d].rdev->nr_pending);
  1327. atomic_inc(&r10_bio->remaining);
  1328. bio->bi_next = biolist;
  1329. biolist = bio;
  1330. bio->bi_private = r10_bio;
  1331. bio->bi_end_io = end_sync_read;
  1332. bio->bi_rw = 0;
  1333. bio->bi_sector = r10_bio->devs[i].addr +
  1334. conf->mirrors[d].rdev->data_offset;
  1335. bio->bi_bdev = conf->mirrors[d].rdev->bdev;
  1336. count++;
  1337. }
  1338. if (count < 2) {
  1339. for (i=0; i<conf->copies; i++) {
  1340. int d = r10_bio->devs[i].devnum;
  1341. if (r10_bio->devs[i].bio->bi_end_io)
  1342. rdev_dec_pending(conf->mirrors[d].rdev, mddev);
  1343. }
  1344. put_buf(r10_bio);
  1345. biolist = NULL;
  1346. goto giveup;
  1347. }
  1348. }
  1349. for (bio = biolist; bio ; bio=bio->bi_next) {
  1350. bio->bi_flags &= ~(BIO_POOL_MASK - 1);
  1351. if (bio->bi_end_io)
  1352. bio->bi_flags |= 1 << BIO_UPTODATE;
  1353. bio->bi_vcnt = 0;
  1354. bio->bi_idx = 0;
  1355. bio->bi_phys_segments = 0;
  1356. bio->bi_hw_segments = 0;
  1357. bio->bi_size = 0;
  1358. }
  1359. nr_sectors = 0;
  1360. do {
  1361. struct page *page;
  1362. int len = PAGE_SIZE;
  1363. disk = 0;
  1364. if (sector_nr + (len>>9) > max_sector)
  1365. len = (max_sector - sector_nr) << 9;
  1366. if (len == 0)
  1367. break;
  1368. for (bio= biolist ; bio ; bio=bio->bi_next) {
  1369. page = bio->bi_io_vec[bio->bi_vcnt].bv_page;
  1370. if (bio_add_page(bio, page, len, 0) == 0) {
  1371. /* stop here */
  1372. struct bio *bio2;
  1373. bio->bi_io_vec[bio->bi_vcnt].bv_page = page;
  1374. for (bio2 = biolist; bio2 && bio2 != bio; bio2 = bio2->bi_next) {
  1375. /* remove last page from this bio */
  1376. bio2->bi_vcnt--;
  1377. bio2->bi_size -= len;
  1378. bio2->bi_flags &= ~(1<< BIO_SEG_VALID);
  1379. }
  1380. goto bio_full;
  1381. }
  1382. disk = i;
  1383. }
  1384. nr_sectors += len>>9;
  1385. sector_nr += len>>9;
  1386. } while (biolist->bi_vcnt < RESYNC_PAGES);
  1387. bio_full:
  1388. r10_bio->sectors = nr_sectors;
  1389. while (biolist) {
  1390. bio = biolist;
  1391. biolist = biolist->bi_next;
  1392. bio->bi_next = NULL;
  1393. r10_bio = bio->bi_private;
  1394. r10_bio->sectors = nr_sectors;
  1395. if (bio->bi_end_io == end_sync_read) {
  1396. md_sync_acct(bio->bi_bdev, nr_sectors);
  1397. generic_make_request(bio);
  1398. }
  1399. }
  1400. if (sectors_skipped)
  1401. /* pretend they weren't skipped, it makes
  1402. * no important difference in this case
  1403. */
  1404. md_done_sync(mddev, sectors_skipped, 1);
  1405. return sectors_skipped + nr_sectors;
  1406. giveup:
  1407. /* There is nowhere to write, so all non-sync
  1408. * drives must be failed, so try the next chunk...
  1409. */
  1410. {
  1411. sector_t sec = max_sector - sector_nr;
  1412. sectors_skipped += sec;
  1413. chunks_skipped ++;
  1414. sector_nr = max_sector;
  1415. goto skipped;
  1416. }
  1417. }
  1418. static int run(mddev_t *mddev)
  1419. {
  1420. conf_t *conf;
  1421. int i, disk_idx;
  1422. mirror_info_t *disk;
  1423. mdk_rdev_t *rdev;
  1424. struct list_head *tmp;
  1425. int nc, fc;
  1426. sector_t stride, size;
  1427. if (mddev->level != 10) {
  1428. printk(KERN_ERR "raid10: %s: raid level not set correctly... (%d)\n",
  1429. mdname(mddev), mddev->level);
  1430. goto out;
  1431. }
  1432. nc = mddev->layout & 255;
  1433. fc = (mddev->layout >> 8) & 255;
  1434. if ((nc*fc) <2 || (nc*fc) > mddev->raid_disks ||
  1435. (mddev->layout >> 16)) {
  1436. printk(KERN_ERR "raid10: %s: unsupported raid10 layout: 0x%8x\n",
  1437. mdname(mddev), mddev->layout);
  1438. goto out;
  1439. }
  1440. /*
  1441. * copy the already verified devices into our private RAID10
  1442. * bookkeeping area. [whatever we allocate in run(),
  1443. * should be freed in stop()]
  1444. */
  1445. conf = kmalloc(sizeof(conf_t), GFP_KERNEL);
  1446. mddev->private = conf;
  1447. if (!conf) {
  1448. printk(KERN_ERR "raid10: couldn't allocate memory for %s\n",
  1449. mdname(mddev));
  1450. goto out;
  1451. }
  1452. memset(conf, 0, sizeof(*conf));
  1453. conf->mirrors = kmalloc(sizeof(struct mirror_info)*mddev->raid_disks,
  1454. GFP_KERNEL);
  1455. if (!conf->mirrors) {
  1456. printk(KERN_ERR "raid10: couldn't allocate memory for %s\n",
  1457. mdname(mddev));
  1458. goto out_free_conf;
  1459. }
  1460. memset(conf->mirrors, 0, sizeof(struct mirror_info)*mddev->raid_disks);
  1461. conf->near_copies = nc;
  1462. conf->far_copies = fc;
  1463. conf->copies = nc*fc;
  1464. conf->chunk_mask = (sector_t)(mddev->chunk_size>>9)-1;
  1465. conf->chunk_shift = ffz(~mddev->chunk_size) - 9;
  1466. stride = mddev->size >> (conf->chunk_shift-1);
  1467. sector_div(stride, fc);
  1468. conf->stride = stride << conf->chunk_shift;
  1469. conf->r10bio_pool = mempool_create(NR_RAID10_BIOS, r10bio_pool_alloc,
  1470. r10bio_pool_free, conf);
  1471. if (!conf->r10bio_pool) {
  1472. printk(KERN_ERR "raid10: couldn't allocate memory for %s\n",
  1473. mdname(mddev));
  1474. goto out_free_conf;
  1475. }
  1476. ITERATE_RDEV(mddev, rdev, tmp) {
  1477. disk_idx = rdev->raid_disk;
  1478. if (disk_idx >= mddev->raid_disks
  1479. || disk_idx < 0)
  1480. continue;
  1481. disk = conf->mirrors + disk_idx;
  1482. disk->rdev = rdev;
  1483. blk_queue_stack_limits(mddev->queue,
  1484. rdev->bdev->bd_disk->queue);
  1485. /* as we don't honour merge_bvec_fn, we must never risk
  1486. * violating it, so limit ->max_sector to one PAGE, as
  1487. * a one page request is never in violation.
  1488. */
  1489. if (rdev->bdev->bd_disk->queue->merge_bvec_fn &&
  1490. mddev->queue->max_sectors > (PAGE_SIZE>>9))
  1491. mddev->queue->max_sectors = (PAGE_SIZE>>9);
  1492. disk->head_position = 0;
  1493. if (!rdev->faulty && rdev->in_sync)
  1494. conf->working_disks++;
  1495. }
  1496. conf->raid_disks = mddev->raid_disks;
  1497. conf->mddev = mddev;
  1498. spin_lock_init(&conf->device_lock);
  1499. INIT_LIST_HEAD(&conf->retry_list);
  1500. spin_lock_init(&conf->resync_lock);
  1501. init_waitqueue_head(&conf->wait_idle);
  1502. init_waitqueue_head(&conf->wait_resume);
  1503. /* need to check that every block has at least one working mirror */
  1504. if (!enough(conf)) {
  1505. printk(KERN_ERR "raid10: not enough operational mirrors for %s\n",
  1506. mdname(mddev));
  1507. goto out_free_conf;
  1508. }
  1509. mddev->degraded = 0;
  1510. for (i = 0; i < conf->raid_disks; i++) {
  1511. disk = conf->mirrors + i;
  1512. if (!disk->rdev) {
  1513. disk->head_position = 0;
  1514. mddev->degraded++;
  1515. }
  1516. }
  1517. mddev->thread = md_register_thread(raid10d, mddev, "%s_raid10");
  1518. if (!mddev->thread) {
  1519. printk(KERN_ERR
  1520. "raid10: couldn't allocate thread for %s\n",
  1521. mdname(mddev));
  1522. goto out_free_conf;
  1523. }
  1524. printk(KERN_INFO
  1525. "raid10: raid set %s active with %d out of %d devices\n",
  1526. mdname(mddev), mddev->raid_disks - mddev->degraded,
  1527. mddev->raid_disks);
  1528. /*
  1529. * Ok, everything is just fine now
  1530. */
  1531. size = conf->stride * conf->raid_disks;
  1532. sector_div(size, conf->near_copies);
  1533. mddev->array_size = size/2;
  1534. mddev->resync_max_sectors = size;
  1535. mddev->queue->unplug_fn = raid10_unplug;
  1536. mddev->queue->issue_flush_fn = raid10_issue_flush;
  1537. /* Calculate max read-ahead size.
  1538. * We need to readahead at least twice a whole stripe....
  1539. * maybe...
  1540. */
  1541. {
  1542. int stripe = conf->raid_disks * mddev->chunk_size / PAGE_CACHE_SIZE;
  1543. stripe /= conf->near_copies;
  1544. if (mddev->queue->backing_dev_info.ra_pages < 2* stripe)
  1545. mddev->queue->backing_dev_info.ra_pages = 2* stripe;
  1546. }
  1547. if (conf->near_copies < mddev->raid_disks)
  1548. blk_queue_merge_bvec(mddev->queue, raid10_mergeable_bvec);
  1549. return 0;
  1550. out_free_conf:
  1551. if (conf->r10bio_pool)
  1552. mempool_destroy(conf->r10bio_pool);
  1553. kfree(conf->mirrors);
  1554. kfree(conf);
  1555. mddev->private = NULL;
  1556. out:
  1557. return -EIO;
  1558. }
  1559. static int stop(mddev_t *mddev)
  1560. {
  1561. conf_t *conf = mddev_to_conf(mddev);
  1562. md_unregister_thread(mddev->thread);
  1563. mddev->thread = NULL;
  1564. blk_sync_queue(mddev->queue); /* the unplug fn references 'conf'*/
  1565. if (conf->r10bio_pool)
  1566. mempool_destroy(conf->r10bio_pool);
  1567. kfree(conf->mirrors);
  1568. kfree(conf);
  1569. mddev->private = NULL;
  1570. return 0;
  1571. }
  1572. static mdk_personality_t raid10_personality =
  1573. {
  1574. .name = "raid10",
  1575. .owner = THIS_MODULE,
  1576. .make_request = make_request,
  1577. .run = run,
  1578. .stop = stop,
  1579. .status = status,
  1580. .error_handler = error,
  1581. .hot_add_disk = raid10_add_disk,
  1582. .hot_remove_disk= raid10_remove_disk,
  1583. .spare_active = raid10_spare_active,
  1584. .sync_request = sync_request,
  1585. };
  1586. static int __init raid_init(void)
  1587. {
  1588. return register_md_personality(RAID10, &raid10_personality);
  1589. }
  1590. static void raid_exit(void)
  1591. {
  1592. unregister_md_personality(RAID10);
  1593. }
  1594. module_init(raid_init);
  1595. module_exit(raid_exit);
  1596. MODULE_LICENSE("GPL");
  1597. MODULE_ALIAS("md-personality-9"); /* RAID10 */