bitmap.c 41 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601
  1. /*
  2. * bitmap.c two-level bitmap (C) Peter T. Breuer (ptb@ot.uc3m.es) 2003
  3. *
  4. * bitmap_create - sets up the bitmap structure
  5. * bitmap_destroy - destroys the bitmap structure
  6. *
  7. * additions, Copyright (C) 2003-2004, Paul Clements, SteelEye Technology, Inc.:
  8. * - added disk storage for bitmap
  9. * - changes to allow various bitmap chunk sizes
  10. */
  11. /*
  12. * Still to do:
  13. *
  14. * flush after percent set rather than just time based. (maybe both).
  15. * wait if count gets too high, wake when it drops to half.
  16. */
  17. #include <linux/module.h>
  18. #include <linux/errno.h>
  19. #include <linux/slab.h>
  20. #include <linux/init.h>
  21. #include <linux/timer.h>
  22. #include <linux/sched.h>
  23. #include <linux/list.h>
  24. #include <linux/file.h>
  25. #include <linux/mount.h>
  26. #include <linux/buffer_head.h>
  27. #include <linux/raid/md.h>
  28. #include <linux/raid/bitmap.h>
  29. /* debug macros */
  30. #define DEBUG 0
  31. #if DEBUG
  32. /* these are for debugging purposes only! */
  33. /* define one and only one of these */
  34. #define INJECT_FAULTS_1 0 /* cause bitmap_alloc_page to fail always */
  35. #define INJECT_FAULTS_2 0 /* cause bitmap file to be kicked when first bit set*/
  36. #define INJECT_FAULTS_3 0 /* treat bitmap file as kicked at init time */
  37. #define INJECT_FAULTS_4 0 /* undef */
  38. #define INJECT_FAULTS_5 0 /* undef */
  39. #define INJECT_FAULTS_6 0
  40. /* if these are defined, the driver will fail! debug only */
  41. #define INJECT_FATAL_FAULT_1 0 /* fail kmalloc, causing bitmap_create to fail */
  42. #define INJECT_FATAL_FAULT_2 0 /* undef */
  43. #define INJECT_FATAL_FAULT_3 0 /* undef */
  44. #endif
  45. //#define DPRINTK PRINTK /* set this NULL to avoid verbose debug output */
  46. #define DPRINTK(x...) do { } while(0)
  47. #ifndef PRINTK
  48. # if DEBUG > 0
  49. # define PRINTK(x...) printk(KERN_DEBUG x)
  50. # else
  51. # define PRINTK(x...)
  52. # endif
  53. #endif
  54. static inline char * bmname(struct bitmap *bitmap)
  55. {
  56. return bitmap->mddev ? mdname(bitmap->mddev) : "mdX";
  57. }
  58. /*
  59. * just a placeholder - calls kmalloc for bitmap pages
  60. */
  61. static unsigned char *bitmap_alloc_page(struct bitmap *bitmap)
  62. {
  63. unsigned char *page;
  64. #ifdef INJECT_FAULTS_1
  65. page = NULL;
  66. #else
  67. page = kmalloc(PAGE_SIZE, GFP_NOIO);
  68. #endif
  69. if (!page)
  70. printk("%s: bitmap_alloc_page FAILED\n", bmname(bitmap));
  71. else
  72. PRINTK("%s: bitmap_alloc_page: allocated page at %p\n",
  73. bmname(bitmap), page);
  74. return page;
  75. }
  76. /*
  77. * for now just a placeholder -- just calls kfree for bitmap pages
  78. */
  79. static void bitmap_free_page(struct bitmap *bitmap, unsigned char *page)
  80. {
  81. PRINTK("%s: bitmap_free_page: free page %p\n", bmname(bitmap), page);
  82. kfree(page);
  83. }
  84. /*
  85. * check a page and, if necessary, allocate it (or hijack it if the alloc fails)
  86. *
  87. * 1) check to see if this page is allocated, if it's not then try to alloc
  88. * 2) if the alloc fails, set the page's hijacked flag so we'll use the
  89. * page pointer directly as a counter
  90. *
  91. * if we find our page, we increment the page's refcount so that it stays
  92. * allocated while we're using it
  93. */
  94. static int bitmap_checkpage(struct bitmap *bitmap, unsigned long page, int create)
  95. {
  96. unsigned char *mappage;
  97. if (page >= bitmap->pages) {
  98. printk(KERN_ALERT
  99. "%s: invalid bitmap page request: %lu (> %lu)\n",
  100. bmname(bitmap), page, bitmap->pages-1);
  101. return -EINVAL;
  102. }
  103. if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */
  104. return 0;
  105. if (bitmap->bp[page].map) /* page is already allocated, just return */
  106. return 0;
  107. if (!create)
  108. return -ENOENT;
  109. spin_unlock_irq(&bitmap->lock);
  110. /* this page has not been allocated yet */
  111. if ((mappage = bitmap_alloc_page(bitmap)) == NULL) {
  112. PRINTK("%s: bitmap map page allocation failed, hijacking\n",
  113. bmname(bitmap));
  114. /* failed - set the hijacked flag so that we can use the
  115. * pointer as a counter */
  116. spin_lock_irq(&bitmap->lock);
  117. if (!bitmap->bp[page].map)
  118. bitmap->bp[page].hijacked = 1;
  119. goto out;
  120. }
  121. /* got a page */
  122. spin_lock_irq(&bitmap->lock);
  123. /* recheck the page */
  124. if (bitmap->bp[page].map || bitmap->bp[page].hijacked) {
  125. /* somebody beat us to getting the page */
  126. bitmap_free_page(bitmap, mappage);
  127. return 0;
  128. }
  129. /* no page was in place and we have one, so install it */
  130. memset(mappage, 0, PAGE_SIZE);
  131. bitmap->bp[page].map = mappage;
  132. bitmap->missing_pages--;
  133. out:
  134. return 0;
  135. }
  136. /* if page is completely empty, put it back on the free list, or dealloc it */
  137. /* if page was hijacked, unmark the flag so it might get alloced next time */
  138. /* Note: lock should be held when calling this */
  139. static void bitmap_checkfree(struct bitmap *bitmap, unsigned long page)
  140. {
  141. char *ptr;
  142. if (bitmap->bp[page].count) /* page is still busy */
  143. return;
  144. /* page is no longer in use, it can be released */
  145. if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */
  146. bitmap->bp[page].hijacked = 0;
  147. bitmap->bp[page].map = NULL;
  148. return;
  149. }
  150. /* normal case, free the page */
  151. #if 0
  152. /* actually ... let's not. We will probably need the page again exactly when
  153. * memory is tight and we are flusing to disk
  154. */
  155. return;
  156. #else
  157. ptr = bitmap->bp[page].map;
  158. bitmap->bp[page].map = NULL;
  159. bitmap->missing_pages++;
  160. bitmap_free_page(bitmap, ptr);
  161. return;
  162. #endif
  163. }
  164. /*
  165. * bitmap file handling - read and write the bitmap file and its superblock
  166. */
  167. /* copy the pathname of a file to a buffer */
  168. char *file_path(struct file *file, char *buf, int count)
  169. {
  170. if (!buf)
  171. return NULL;
  172. buf = d_path(&file->f_path, buf, count);
  173. return IS_ERR(buf) ? NULL : buf;
  174. }
  175. /*
  176. * basic page I/O operations
  177. */
  178. /* IO operations when bitmap is stored near all superblocks */
  179. static struct page *read_sb_page(mddev_t *mddev, long offset, unsigned long index)
  180. {
  181. /* choose a good rdev and read the page from there */
  182. mdk_rdev_t *rdev;
  183. struct list_head *tmp;
  184. struct page *page = alloc_page(GFP_KERNEL);
  185. sector_t target;
  186. if (!page)
  187. return ERR_PTR(-ENOMEM);
  188. rdev_for_each(rdev, tmp, mddev) {
  189. if (! test_bit(In_sync, &rdev->flags)
  190. || test_bit(Faulty, &rdev->flags))
  191. continue;
  192. target = (rdev->sb_offset << 1) + offset + index * (PAGE_SIZE/512);
  193. if (sync_page_io(rdev->bdev, target, PAGE_SIZE, page, READ)) {
  194. page->index = index;
  195. attach_page_buffers(page, NULL); /* so that free_buffer will
  196. * quietly no-op */
  197. return page;
  198. }
  199. }
  200. return ERR_PTR(-EIO);
  201. }
  202. static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait)
  203. {
  204. mdk_rdev_t *rdev;
  205. struct list_head *tmp;
  206. mddev_t *mddev = bitmap->mddev;
  207. rdev_for_each(rdev, tmp, mddev)
  208. if (test_bit(In_sync, &rdev->flags)
  209. && !test_bit(Faulty, &rdev->flags)) {
  210. int size = PAGE_SIZE;
  211. if (page->index == bitmap->file_pages-1)
  212. size = roundup(bitmap->last_page_size,
  213. bdev_hardsect_size(rdev->bdev));
  214. /* Just make sure we aren't corrupting data or
  215. * metadata
  216. */
  217. if (bitmap->offset < 0) {
  218. /* DATA BITMAP METADATA */
  219. if (bitmap->offset
  220. + (long)(page->index * (PAGE_SIZE/512))
  221. + size/512 > 0)
  222. /* bitmap runs in to metadata */
  223. return -EINVAL;
  224. if (rdev->data_offset + mddev->size*2
  225. > rdev->sb_offset*2 + bitmap->offset)
  226. /* data runs in to bitmap */
  227. return -EINVAL;
  228. } else if (rdev->sb_offset*2 < rdev->data_offset) {
  229. /* METADATA BITMAP DATA */
  230. if (rdev->sb_offset*2
  231. + bitmap->offset
  232. + page->index*(PAGE_SIZE/512) + size/512
  233. > rdev->data_offset)
  234. /* bitmap runs in to data */
  235. return -EINVAL;
  236. } else {
  237. /* DATA METADATA BITMAP - no problems */
  238. }
  239. md_super_write(mddev, rdev,
  240. (rdev->sb_offset<<1) + bitmap->offset
  241. + page->index * (PAGE_SIZE/512),
  242. size,
  243. page);
  244. }
  245. if (wait)
  246. md_super_wait(mddev);
  247. return 0;
  248. }
  249. static void bitmap_file_kick(struct bitmap *bitmap);
  250. /*
  251. * write out a page to a file
  252. */
  253. static void write_page(struct bitmap *bitmap, struct page *page, int wait)
  254. {
  255. struct buffer_head *bh;
  256. if (bitmap->file == NULL) {
  257. switch (write_sb_page(bitmap, page, wait)) {
  258. case -EINVAL:
  259. bitmap->flags |= BITMAP_WRITE_ERROR;
  260. }
  261. } else {
  262. bh = page_buffers(page);
  263. while (bh && bh->b_blocknr) {
  264. atomic_inc(&bitmap->pending_writes);
  265. set_buffer_locked(bh);
  266. set_buffer_mapped(bh);
  267. submit_bh(WRITE, bh);
  268. bh = bh->b_this_page;
  269. }
  270. if (wait) {
  271. wait_event(bitmap->write_wait,
  272. atomic_read(&bitmap->pending_writes)==0);
  273. }
  274. }
  275. if (bitmap->flags & BITMAP_WRITE_ERROR)
  276. bitmap_file_kick(bitmap);
  277. }
  278. static void end_bitmap_write(struct buffer_head *bh, int uptodate)
  279. {
  280. struct bitmap *bitmap = bh->b_private;
  281. unsigned long flags;
  282. if (!uptodate) {
  283. spin_lock_irqsave(&bitmap->lock, flags);
  284. bitmap->flags |= BITMAP_WRITE_ERROR;
  285. spin_unlock_irqrestore(&bitmap->lock, flags);
  286. }
  287. if (atomic_dec_and_test(&bitmap->pending_writes))
  288. wake_up(&bitmap->write_wait);
  289. }
  290. /* copied from buffer.c */
  291. static void
  292. __clear_page_buffers(struct page *page)
  293. {
  294. ClearPagePrivate(page);
  295. set_page_private(page, 0);
  296. page_cache_release(page);
  297. }
  298. static void free_buffers(struct page *page)
  299. {
  300. struct buffer_head *bh = page_buffers(page);
  301. while (bh) {
  302. struct buffer_head *next = bh->b_this_page;
  303. free_buffer_head(bh);
  304. bh = next;
  305. }
  306. __clear_page_buffers(page);
  307. put_page(page);
  308. }
  309. /* read a page from a file.
  310. * We both read the page, and attach buffers to the page to record the
  311. * address of each block (using bmap). These addresses will be used
  312. * to write the block later, completely bypassing the filesystem.
  313. * This usage is similar to how swap files are handled, and allows us
  314. * to write to a file with no concerns of memory allocation failing.
  315. */
  316. static struct page *read_page(struct file *file, unsigned long index,
  317. struct bitmap *bitmap,
  318. unsigned long count)
  319. {
  320. struct page *page = NULL;
  321. struct inode *inode = file->f_path.dentry->d_inode;
  322. struct buffer_head *bh;
  323. sector_t block;
  324. PRINTK("read bitmap file (%dB @ %Lu)\n", (int)PAGE_SIZE,
  325. (unsigned long long)index << PAGE_SHIFT);
  326. page = alloc_page(GFP_KERNEL);
  327. if (!page)
  328. page = ERR_PTR(-ENOMEM);
  329. if (IS_ERR(page))
  330. goto out;
  331. bh = alloc_page_buffers(page, 1<<inode->i_blkbits, 0);
  332. if (!bh) {
  333. put_page(page);
  334. page = ERR_PTR(-ENOMEM);
  335. goto out;
  336. }
  337. attach_page_buffers(page, bh);
  338. block = index << (PAGE_SHIFT - inode->i_blkbits);
  339. while (bh) {
  340. if (count == 0)
  341. bh->b_blocknr = 0;
  342. else {
  343. bh->b_blocknr = bmap(inode, block);
  344. if (bh->b_blocknr == 0) {
  345. /* Cannot use this file! */
  346. free_buffers(page);
  347. page = ERR_PTR(-EINVAL);
  348. goto out;
  349. }
  350. bh->b_bdev = inode->i_sb->s_bdev;
  351. if (count < (1<<inode->i_blkbits))
  352. count = 0;
  353. else
  354. count -= (1<<inode->i_blkbits);
  355. bh->b_end_io = end_bitmap_write;
  356. bh->b_private = bitmap;
  357. atomic_inc(&bitmap->pending_writes);
  358. set_buffer_locked(bh);
  359. set_buffer_mapped(bh);
  360. submit_bh(READ, bh);
  361. }
  362. block++;
  363. bh = bh->b_this_page;
  364. }
  365. page->index = index;
  366. wait_event(bitmap->write_wait,
  367. atomic_read(&bitmap->pending_writes)==0);
  368. if (bitmap->flags & BITMAP_WRITE_ERROR) {
  369. free_buffers(page);
  370. page = ERR_PTR(-EIO);
  371. }
  372. out:
  373. if (IS_ERR(page))
  374. printk(KERN_ALERT "md: bitmap read error: (%dB @ %Lu): %ld\n",
  375. (int)PAGE_SIZE,
  376. (unsigned long long)index << PAGE_SHIFT,
  377. PTR_ERR(page));
  378. return page;
  379. }
  380. /*
  381. * bitmap file superblock operations
  382. */
  383. /* update the event counter and sync the superblock to disk */
  384. void bitmap_update_sb(struct bitmap *bitmap)
  385. {
  386. bitmap_super_t *sb;
  387. unsigned long flags;
  388. if (!bitmap || !bitmap->mddev) /* no bitmap for this array */
  389. return;
  390. spin_lock_irqsave(&bitmap->lock, flags);
  391. if (!bitmap->sb_page) { /* no superblock */
  392. spin_unlock_irqrestore(&bitmap->lock, flags);
  393. return;
  394. }
  395. spin_unlock_irqrestore(&bitmap->lock, flags);
  396. sb = (bitmap_super_t *)kmap_atomic(bitmap->sb_page, KM_USER0);
  397. sb->events = cpu_to_le64(bitmap->mddev->events);
  398. if (!bitmap->mddev->degraded)
  399. sb->events_cleared = cpu_to_le64(bitmap->mddev->events);
  400. kunmap_atomic(sb, KM_USER0);
  401. write_page(bitmap, bitmap->sb_page, 1);
  402. }
  403. /* print out the bitmap file superblock */
  404. void bitmap_print_sb(struct bitmap *bitmap)
  405. {
  406. bitmap_super_t *sb;
  407. if (!bitmap || !bitmap->sb_page)
  408. return;
  409. sb = (bitmap_super_t *)kmap_atomic(bitmap->sb_page, KM_USER0);
  410. printk(KERN_DEBUG "%s: bitmap file superblock:\n", bmname(bitmap));
  411. printk(KERN_DEBUG " magic: %08x\n", le32_to_cpu(sb->magic));
  412. printk(KERN_DEBUG " version: %d\n", le32_to_cpu(sb->version));
  413. printk(KERN_DEBUG " uuid: %08x.%08x.%08x.%08x\n",
  414. *(__u32 *)(sb->uuid+0),
  415. *(__u32 *)(sb->uuid+4),
  416. *(__u32 *)(sb->uuid+8),
  417. *(__u32 *)(sb->uuid+12));
  418. printk(KERN_DEBUG " events: %llu\n",
  419. (unsigned long long) le64_to_cpu(sb->events));
  420. printk(KERN_DEBUG "events cleared: %llu\n",
  421. (unsigned long long) le64_to_cpu(sb->events_cleared));
  422. printk(KERN_DEBUG " state: %08x\n", le32_to_cpu(sb->state));
  423. printk(KERN_DEBUG " chunksize: %d B\n", le32_to_cpu(sb->chunksize));
  424. printk(KERN_DEBUG " daemon sleep: %ds\n", le32_to_cpu(sb->daemon_sleep));
  425. printk(KERN_DEBUG " sync size: %llu KB\n",
  426. (unsigned long long)le64_to_cpu(sb->sync_size)/2);
  427. printk(KERN_DEBUG "max write behind: %d\n", le32_to_cpu(sb->write_behind));
  428. kunmap_atomic(sb, KM_USER0);
  429. }
  430. /* read the superblock from the bitmap file and initialize some bitmap fields */
  431. static int bitmap_read_sb(struct bitmap *bitmap)
  432. {
  433. char *reason = NULL;
  434. bitmap_super_t *sb;
  435. unsigned long chunksize, daemon_sleep, write_behind;
  436. unsigned long long events;
  437. int err = -EINVAL;
  438. /* page 0 is the superblock, read it... */
  439. if (bitmap->file) {
  440. loff_t isize = i_size_read(bitmap->file->f_mapping->host);
  441. int bytes = isize > PAGE_SIZE ? PAGE_SIZE : isize;
  442. bitmap->sb_page = read_page(bitmap->file, 0, bitmap, bytes);
  443. } else {
  444. bitmap->sb_page = read_sb_page(bitmap->mddev, bitmap->offset, 0);
  445. }
  446. if (IS_ERR(bitmap->sb_page)) {
  447. err = PTR_ERR(bitmap->sb_page);
  448. bitmap->sb_page = NULL;
  449. return err;
  450. }
  451. sb = (bitmap_super_t *)kmap_atomic(bitmap->sb_page, KM_USER0);
  452. chunksize = le32_to_cpu(sb->chunksize);
  453. daemon_sleep = le32_to_cpu(sb->daemon_sleep);
  454. write_behind = le32_to_cpu(sb->write_behind);
  455. /* verify that the bitmap-specific fields are valid */
  456. if (sb->magic != cpu_to_le32(BITMAP_MAGIC))
  457. reason = "bad magic";
  458. else if (le32_to_cpu(sb->version) < BITMAP_MAJOR_LO ||
  459. le32_to_cpu(sb->version) > BITMAP_MAJOR_HI)
  460. reason = "unrecognized superblock version";
  461. else if (chunksize < PAGE_SIZE)
  462. reason = "bitmap chunksize too small";
  463. else if ((1 << ffz(~chunksize)) != chunksize)
  464. reason = "bitmap chunksize not a power of 2";
  465. else if (daemon_sleep < 1 || daemon_sleep > MAX_SCHEDULE_TIMEOUT / HZ)
  466. reason = "daemon sleep period out of range";
  467. else if (write_behind > COUNTER_MAX)
  468. reason = "write-behind limit out of range (0 - 16383)";
  469. if (reason) {
  470. printk(KERN_INFO "%s: invalid bitmap file superblock: %s\n",
  471. bmname(bitmap), reason);
  472. goto out;
  473. }
  474. /* keep the array size field of the bitmap superblock up to date */
  475. sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
  476. if (!bitmap->mddev->persistent)
  477. goto success;
  478. /*
  479. * if we have a persistent array superblock, compare the
  480. * bitmap's UUID and event counter to the mddev's
  481. */
  482. if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) {
  483. printk(KERN_INFO "%s: bitmap superblock UUID mismatch\n",
  484. bmname(bitmap));
  485. goto out;
  486. }
  487. events = le64_to_cpu(sb->events);
  488. if (events < bitmap->mddev->events) {
  489. printk(KERN_INFO "%s: bitmap file is out of date (%llu < %llu) "
  490. "-- forcing full recovery\n", bmname(bitmap), events,
  491. (unsigned long long) bitmap->mddev->events);
  492. sb->state |= cpu_to_le32(BITMAP_STALE);
  493. }
  494. success:
  495. /* assign fields using values from superblock */
  496. bitmap->chunksize = chunksize;
  497. bitmap->daemon_sleep = daemon_sleep;
  498. bitmap->daemon_lastrun = jiffies;
  499. bitmap->max_write_behind = write_behind;
  500. bitmap->flags |= le32_to_cpu(sb->state);
  501. if (le32_to_cpu(sb->version) == BITMAP_MAJOR_HOSTENDIAN)
  502. bitmap->flags |= BITMAP_HOSTENDIAN;
  503. bitmap->events_cleared = le64_to_cpu(sb->events_cleared);
  504. if (sb->state & cpu_to_le32(BITMAP_STALE))
  505. bitmap->events_cleared = bitmap->mddev->events;
  506. err = 0;
  507. out:
  508. kunmap_atomic(sb, KM_USER0);
  509. if (err)
  510. bitmap_print_sb(bitmap);
  511. return err;
  512. }
  513. enum bitmap_mask_op {
  514. MASK_SET,
  515. MASK_UNSET
  516. };
  517. /* record the state of the bitmap in the superblock. Return the old value */
  518. static int bitmap_mask_state(struct bitmap *bitmap, enum bitmap_state bits,
  519. enum bitmap_mask_op op)
  520. {
  521. bitmap_super_t *sb;
  522. unsigned long flags;
  523. int old;
  524. spin_lock_irqsave(&bitmap->lock, flags);
  525. if (!bitmap->sb_page) { /* can't set the state */
  526. spin_unlock_irqrestore(&bitmap->lock, flags);
  527. return 0;
  528. }
  529. spin_unlock_irqrestore(&bitmap->lock, flags);
  530. sb = (bitmap_super_t *)kmap_atomic(bitmap->sb_page, KM_USER0);
  531. old = le32_to_cpu(sb->state) & bits;
  532. switch (op) {
  533. case MASK_SET: sb->state |= cpu_to_le32(bits);
  534. break;
  535. case MASK_UNSET: sb->state &= cpu_to_le32(~bits);
  536. break;
  537. default: BUG();
  538. }
  539. kunmap_atomic(sb, KM_USER0);
  540. return old;
  541. }
  542. /*
  543. * general bitmap file operations
  544. */
  545. /* calculate the index of the page that contains this bit */
  546. static inline unsigned long file_page_index(unsigned long chunk)
  547. {
  548. return CHUNK_BIT_OFFSET(chunk) >> PAGE_BIT_SHIFT;
  549. }
  550. /* calculate the (bit) offset of this bit within a page */
  551. static inline unsigned long file_page_offset(unsigned long chunk)
  552. {
  553. return CHUNK_BIT_OFFSET(chunk) & (PAGE_BITS - 1);
  554. }
  555. /*
  556. * return a pointer to the page in the filemap that contains the given bit
  557. *
  558. * this lookup is complicated by the fact that the bitmap sb might be exactly
  559. * 1 page (e.g., x86) or less than 1 page -- so the bitmap might start on page
  560. * 0 or page 1
  561. */
  562. static inline struct page *filemap_get_page(struct bitmap *bitmap,
  563. unsigned long chunk)
  564. {
  565. if (file_page_index(chunk) >= bitmap->file_pages) return NULL;
  566. return bitmap->filemap[file_page_index(chunk) - file_page_index(0)];
  567. }
  568. static void bitmap_file_unmap(struct bitmap *bitmap)
  569. {
  570. struct page **map, *sb_page;
  571. unsigned long *attr;
  572. int pages;
  573. unsigned long flags;
  574. spin_lock_irqsave(&bitmap->lock, flags);
  575. map = bitmap->filemap;
  576. bitmap->filemap = NULL;
  577. attr = bitmap->filemap_attr;
  578. bitmap->filemap_attr = NULL;
  579. pages = bitmap->file_pages;
  580. bitmap->file_pages = 0;
  581. sb_page = bitmap->sb_page;
  582. bitmap->sb_page = NULL;
  583. spin_unlock_irqrestore(&bitmap->lock, flags);
  584. while (pages--)
  585. if (map[pages]->index != 0) /* 0 is sb_page, release it below */
  586. free_buffers(map[pages]);
  587. kfree(map);
  588. kfree(attr);
  589. if (sb_page)
  590. free_buffers(sb_page);
  591. }
  592. static void bitmap_file_put(struct bitmap *bitmap)
  593. {
  594. struct file *file;
  595. unsigned long flags;
  596. spin_lock_irqsave(&bitmap->lock, flags);
  597. file = bitmap->file;
  598. bitmap->file = NULL;
  599. spin_unlock_irqrestore(&bitmap->lock, flags);
  600. if (file)
  601. wait_event(bitmap->write_wait,
  602. atomic_read(&bitmap->pending_writes)==0);
  603. bitmap_file_unmap(bitmap);
  604. if (file) {
  605. struct inode *inode = file->f_path.dentry->d_inode;
  606. invalidate_mapping_pages(inode->i_mapping, 0, -1);
  607. fput(file);
  608. }
  609. }
  610. /*
  611. * bitmap_file_kick - if an error occurs while manipulating the bitmap file
  612. * then it is no longer reliable, so we stop using it and we mark the file
  613. * as failed in the superblock
  614. */
  615. static void bitmap_file_kick(struct bitmap *bitmap)
  616. {
  617. char *path, *ptr = NULL;
  618. if (bitmap_mask_state(bitmap, BITMAP_STALE, MASK_SET) == 0) {
  619. bitmap_update_sb(bitmap);
  620. if (bitmap->file) {
  621. path = kmalloc(PAGE_SIZE, GFP_KERNEL);
  622. if (path)
  623. ptr = file_path(bitmap->file, path, PAGE_SIZE);
  624. printk(KERN_ALERT
  625. "%s: kicking failed bitmap file %s from array!\n",
  626. bmname(bitmap), ptr ? ptr : "");
  627. kfree(path);
  628. } else
  629. printk(KERN_ALERT
  630. "%s: disabling internal bitmap due to errors\n",
  631. bmname(bitmap));
  632. }
  633. bitmap_file_put(bitmap);
  634. return;
  635. }
  636. enum bitmap_page_attr {
  637. BITMAP_PAGE_DIRTY = 0, // there are set bits that need to be synced
  638. BITMAP_PAGE_CLEAN = 1, // there are bits that might need to be cleared
  639. BITMAP_PAGE_NEEDWRITE=2, // there are cleared bits that need to be synced
  640. };
  641. static inline void set_page_attr(struct bitmap *bitmap, struct page *page,
  642. enum bitmap_page_attr attr)
  643. {
  644. __set_bit((page->index<<2) + attr, bitmap->filemap_attr);
  645. }
  646. static inline void clear_page_attr(struct bitmap *bitmap, struct page *page,
  647. enum bitmap_page_attr attr)
  648. {
  649. __clear_bit((page->index<<2) + attr, bitmap->filemap_attr);
  650. }
  651. static inline unsigned long test_page_attr(struct bitmap *bitmap, struct page *page,
  652. enum bitmap_page_attr attr)
  653. {
  654. return test_bit((page->index<<2) + attr, bitmap->filemap_attr);
  655. }
  656. /*
  657. * bitmap_file_set_bit -- called before performing a write to the md device
  658. * to set (and eventually sync) a particular bit in the bitmap file
  659. *
  660. * we set the bit immediately, then we record the page number so that
  661. * when an unplug occurs, we can flush the dirty pages out to disk
  662. */
  663. static void bitmap_file_set_bit(struct bitmap *bitmap, sector_t block)
  664. {
  665. unsigned long bit;
  666. struct page *page;
  667. void *kaddr;
  668. unsigned long chunk = block >> CHUNK_BLOCK_SHIFT(bitmap);
  669. if (!bitmap->filemap) {
  670. return;
  671. }
  672. page = filemap_get_page(bitmap, chunk);
  673. if (!page) return;
  674. bit = file_page_offset(chunk);
  675. /* set the bit */
  676. kaddr = kmap_atomic(page, KM_USER0);
  677. if (bitmap->flags & BITMAP_HOSTENDIAN)
  678. set_bit(bit, kaddr);
  679. else
  680. ext2_set_bit(bit, kaddr);
  681. kunmap_atomic(kaddr, KM_USER0);
  682. PRINTK("set file bit %lu page %lu\n", bit, page->index);
  683. /* record page number so it gets flushed to disk when unplug occurs */
  684. set_page_attr(bitmap, page, BITMAP_PAGE_DIRTY);
  685. }
  686. /* this gets called when the md device is ready to unplug its underlying
  687. * (slave) device queues -- before we let any writes go down, we need to
  688. * sync the dirty pages of the bitmap file to disk */
  689. void bitmap_unplug(struct bitmap *bitmap)
  690. {
  691. unsigned long i, flags;
  692. int dirty, need_write;
  693. struct page *page;
  694. int wait = 0;
  695. if (!bitmap)
  696. return;
  697. /* look at each page to see if there are any set bits that need to be
  698. * flushed out to disk */
  699. for (i = 0; i < bitmap->file_pages; i++) {
  700. spin_lock_irqsave(&bitmap->lock, flags);
  701. if (!bitmap->filemap) {
  702. spin_unlock_irqrestore(&bitmap->lock, flags);
  703. return;
  704. }
  705. page = bitmap->filemap[i];
  706. dirty = test_page_attr(bitmap, page, BITMAP_PAGE_DIRTY);
  707. need_write = test_page_attr(bitmap, page, BITMAP_PAGE_NEEDWRITE);
  708. clear_page_attr(bitmap, page, BITMAP_PAGE_DIRTY);
  709. clear_page_attr(bitmap, page, BITMAP_PAGE_NEEDWRITE);
  710. if (dirty)
  711. wait = 1;
  712. spin_unlock_irqrestore(&bitmap->lock, flags);
  713. if (dirty | need_write)
  714. write_page(bitmap, page, 0);
  715. }
  716. if (wait) { /* if any writes were performed, we need to wait on them */
  717. if (bitmap->file)
  718. wait_event(bitmap->write_wait,
  719. atomic_read(&bitmap->pending_writes)==0);
  720. else
  721. md_super_wait(bitmap->mddev);
  722. }
  723. if (bitmap->flags & BITMAP_WRITE_ERROR)
  724. bitmap_file_kick(bitmap);
  725. }
  726. static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);
  727. /* * bitmap_init_from_disk -- called at bitmap_create time to initialize
  728. * the in-memory bitmap from the on-disk bitmap -- also, sets up the
  729. * memory mapping of the bitmap file
  730. * Special cases:
  731. * if there's no bitmap file, or if the bitmap file had been
  732. * previously kicked from the array, we mark all the bits as
  733. * 1's in order to cause a full resync.
  734. *
  735. * We ignore all bits for sectors that end earlier than 'start'.
  736. * This is used when reading an out-of-date bitmap...
  737. */
  738. static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
  739. {
  740. unsigned long i, chunks, index, oldindex, bit;
  741. struct page *page = NULL, *oldpage = NULL;
  742. unsigned long num_pages, bit_cnt = 0;
  743. struct file *file;
  744. unsigned long bytes, offset;
  745. int outofdate;
  746. int ret = -ENOSPC;
  747. void *paddr;
  748. chunks = bitmap->chunks;
  749. file = bitmap->file;
  750. BUG_ON(!file && !bitmap->offset);
  751. #ifdef INJECT_FAULTS_3
  752. outofdate = 1;
  753. #else
  754. outofdate = bitmap->flags & BITMAP_STALE;
  755. #endif
  756. if (outofdate)
  757. printk(KERN_INFO "%s: bitmap file is out of date, doing full "
  758. "recovery\n", bmname(bitmap));
  759. bytes = (chunks + 7) / 8;
  760. num_pages = (bytes + sizeof(bitmap_super_t) + PAGE_SIZE - 1) / PAGE_SIZE;
  761. if (file && i_size_read(file->f_mapping->host) < bytes + sizeof(bitmap_super_t)) {
  762. printk(KERN_INFO "%s: bitmap file too short %lu < %lu\n",
  763. bmname(bitmap),
  764. (unsigned long) i_size_read(file->f_mapping->host),
  765. bytes + sizeof(bitmap_super_t));
  766. goto err;
  767. }
  768. ret = -ENOMEM;
  769. bitmap->filemap = kmalloc(sizeof(struct page *) * num_pages, GFP_KERNEL);
  770. if (!bitmap->filemap)
  771. goto err;
  772. /* We need 4 bits per page, rounded up to a multiple of sizeof(unsigned long) */
  773. bitmap->filemap_attr = kzalloc(
  774. roundup( DIV_ROUND_UP(num_pages*4, 8), sizeof(unsigned long)),
  775. GFP_KERNEL);
  776. if (!bitmap->filemap_attr)
  777. goto err;
  778. oldindex = ~0L;
  779. for (i = 0; i < chunks; i++) {
  780. int b;
  781. index = file_page_index(i);
  782. bit = file_page_offset(i);
  783. if (index != oldindex) { /* this is a new page, read it in */
  784. int count;
  785. /* unmap the old page, we're done with it */
  786. if (index == num_pages-1)
  787. count = bytes + sizeof(bitmap_super_t)
  788. - index * PAGE_SIZE;
  789. else
  790. count = PAGE_SIZE;
  791. if (index == 0) {
  792. /*
  793. * if we're here then the superblock page
  794. * contains some bits (PAGE_SIZE != sizeof sb)
  795. * we've already read it in, so just use it
  796. */
  797. page = bitmap->sb_page;
  798. offset = sizeof(bitmap_super_t);
  799. } else if (file) {
  800. page = read_page(file, index, bitmap, count);
  801. offset = 0;
  802. } else {
  803. page = read_sb_page(bitmap->mddev, bitmap->offset, index);
  804. offset = 0;
  805. }
  806. if (IS_ERR(page)) { /* read error */
  807. ret = PTR_ERR(page);
  808. goto err;
  809. }
  810. oldindex = index;
  811. oldpage = page;
  812. if (outofdate) {
  813. /*
  814. * if bitmap is out of date, dirty the
  815. * whole page and write it out
  816. */
  817. paddr = kmap_atomic(page, KM_USER0);
  818. memset(paddr + offset, 0xff,
  819. PAGE_SIZE - offset);
  820. kunmap_atomic(paddr, KM_USER0);
  821. write_page(bitmap, page, 1);
  822. ret = -EIO;
  823. if (bitmap->flags & BITMAP_WRITE_ERROR) {
  824. /* release, page not in filemap yet */
  825. put_page(page);
  826. goto err;
  827. }
  828. }
  829. bitmap->filemap[bitmap->file_pages++] = page;
  830. bitmap->last_page_size = count;
  831. }
  832. paddr = kmap_atomic(page, KM_USER0);
  833. if (bitmap->flags & BITMAP_HOSTENDIAN)
  834. b = test_bit(bit, paddr);
  835. else
  836. b = ext2_test_bit(bit, paddr);
  837. kunmap_atomic(paddr, KM_USER0);
  838. if (b) {
  839. /* if the disk bit is set, set the memory bit */
  840. bitmap_set_memory_bits(bitmap, i << CHUNK_BLOCK_SHIFT(bitmap),
  841. ((i+1) << (CHUNK_BLOCK_SHIFT(bitmap)) >= start)
  842. );
  843. bit_cnt++;
  844. set_page_attr(bitmap, page, BITMAP_PAGE_CLEAN);
  845. }
  846. }
  847. /* everything went OK */
  848. ret = 0;
  849. bitmap_mask_state(bitmap, BITMAP_STALE, MASK_UNSET);
  850. if (bit_cnt) { /* Kick recovery if any bits were set */
  851. set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery);
  852. md_wakeup_thread(bitmap->mddev->thread);
  853. }
  854. printk(KERN_INFO "%s: bitmap initialized from disk: "
  855. "read %lu/%lu pages, set %lu bits\n",
  856. bmname(bitmap), bitmap->file_pages, num_pages, bit_cnt);
  857. return 0;
  858. err:
  859. printk(KERN_INFO "%s: bitmap initialisation failed: %d\n",
  860. bmname(bitmap), ret);
  861. return ret;
  862. }
  863. void bitmap_write_all(struct bitmap *bitmap)
  864. {
  865. /* We don't actually write all bitmap blocks here,
  866. * just flag them as needing to be written
  867. */
  868. int i;
  869. for (i=0; i < bitmap->file_pages; i++)
  870. set_page_attr(bitmap, bitmap->filemap[i],
  871. BITMAP_PAGE_NEEDWRITE);
  872. }
  873. static void bitmap_count_page(struct bitmap *bitmap, sector_t offset, int inc)
  874. {
  875. sector_t chunk = offset >> CHUNK_BLOCK_SHIFT(bitmap);
  876. unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
  877. bitmap->bp[page].count += inc;
  878. /*
  879. if (page == 0) printk("count page 0, offset %llu: %d gives %d\n",
  880. (unsigned long long)offset, inc, bitmap->bp[page].count);
  881. */
  882. bitmap_checkfree(bitmap, page);
  883. }
  884. static bitmap_counter_t *bitmap_get_counter(struct bitmap *bitmap,
  885. sector_t offset, int *blocks,
  886. int create);
  887. /*
  888. * bitmap daemon -- periodically wakes up to clean bits and flush pages
  889. * out to disk
  890. */
  891. void bitmap_daemon_work(struct bitmap *bitmap)
  892. {
  893. unsigned long j;
  894. unsigned long flags;
  895. struct page *page = NULL, *lastpage = NULL;
  896. int blocks;
  897. void *paddr;
  898. if (bitmap == NULL)
  899. return;
  900. if (time_before(jiffies, bitmap->daemon_lastrun + bitmap->daemon_sleep*HZ))
  901. return;
  902. bitmap->daemon_lastrun = jiffies;
  903. if (bitmap->allclean) {
  904. bitmap->mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
  905. return;
  906. }
  907. bitmap->allclean = 1;
  908. for (j = 0; j < bitmap->chunks; j++) {
  909. bitmap_counter_t *bmc;
  910. spin_lock_irqsave(&bitmap->lock, flags);
  911. if (!bitmap->filemap) {
  912. /* error or shutdown */
  913. spin_unlock_irqrestore(&bitmap->lock, flags);
  914. break;
  915. }
  916. page = filemap_get_page(bitmap, j);
  917. if (page != lastpage) {
  918. /* skip this page unless it's marked as needing cleaning */
  919. if (!test_page_attr(bitmap, page, BITMAP_PAGE_CLEAN)) {
  920. int need_write = test_page_attr(bitmap, page,
  921. BITMAP_PAGE_NEEDWRITE);
  922. if (need_write)
  923. clear_page_attr(bitmap, page, BITMAP_PAGE_NEEDWRITE);
  924. spin_unlock_irqrestore(&bitmap->lock, flags);
  925. if (need_write) {
  926. write_page(bitmap, page, 0);
  927. bitmap->allclean = 0;
  928. }
  929. continue;
  930. }
  931. /* grab the new page, sync and release the old */
  932. if (lastpage != NULL) {
  933. if (test_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE)) {
  934. clear_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE);
  935. spin_unlock_irqrestore(&bitmap->lock, flags);
  936. write_page(bitmap, lastpage, 0);
  937. } else {
  938. set_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE);
  939. spin_unlock_irqrestore(&bitmap->lock, flags);
  940. }
  941. } else
  942. spin_unlock_irqrestore(&bitmap->lock, flags);
  943. lastpage = page;
  944. /*
  945. printk("bitmap clean at page %lu\n", j);
  946. */
  947. spin_lock_irqsave(&bitmap->lock, flags);
  948. clear_page_attr(bitmap, page, BITMAP_PAGE_CLEAN);
  949. }
  950. bmc = bitmap_get_counter(bitmap, j << CHUNK_BLOCK_SHIFT(bitmap),
  951. &blocks, 0);
  952. if (bmc) {
  953. /*
  954. if (j < 100) printk("bitmap: j=%lu, *bmc = 0x%x\n", j, *bmc);
  955. */
  956. if (*bmc)
  957. bitmap->allclean = 0;
  958. if (*bmc == 2) {
  959. *bmc=1; /* maybe clear the bit next time */
  960. set_page_attr(bitmap, page, BITMAP_PAGE_CLEAN);
  961. } else if (*bmc == 1) {
  962. /* we can clear the bit */
  963. *bmc = 0;
  964. bitmap_count_page(bitmap, j << CHUNK_BLOCK_SHIFT(bitmap),
  965. -1);
  966. /* clear the bit */
  967. paddr = kmap_atomic(page, KM_USER0);
  968. if (bitmap->flags & BITMAP_HOSTENDIAN)
  969. clear_bit(file_page_offset(j), paddr);
  970. else
  971. ext2_clear_bit(file_page_offset(j), paddr);
  972. kunmap_atomic(paddr, KM_USER0);
  973. }
  974. }
  975. spin_unlock_irqrestore(&bitmap->lock, flags);
  976. }
  977. /* now sync the final page */
  978. if (lastpage != NULL) {
  979. spin_lock_irqsave(&bitmap->lock, flags);
  980. if (test_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE)) {
  981. clear_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE);
  982. spin_unlock_irqrestore(&bitmap->lock, flags);
  983. write_page(bitmap, lastpage, 0);
  984. } else {
  985. set_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE);
  986. spin_unlock_irqrestore(&bitmap->lock, flags);
  987. }
  988. }
  989. if (bitmap->allclean == 0)
  990. bitmap->mddev->thread->timeout = bitmap->daemon_sleep * HZ;
  991. }
  992. static bitmap_counter_t *bitmap_get_counter(struct bitmap *bitmap,
  993. sector_t offset, int *blocks,
  994. int create)
  995. {
  996. /* If 'create', we might release the lock and reclaim it.
  997. * The lock must have been taken with interrupts enabled.
  998. * If !create, we don't release the lock.
  999. */
  1000. sector_t chunk = offset >> CHUNK_BLOCK_SHIFT(bitmap);
  1001. unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
  1002. unsigned long pageoff = (chunk & PAGE_COUNTER_MASK) << COUNTER_BYTE_SHIFT;
  1003. sector_t csize;
  1004. if (bitmap_checkpage(bitmap, page, create) < 0) {
  1005. csize = ((sector_t)1) << (CHUNK_BLOCK_SHIFT(bitmap));
  1006. *blocks = csize - (offset & (csize- 1));
  1007. return NULL;
  1008. }
  1009. /* now locked ... */
  1010. if (bitmap->bp[page].hijacked) { /* hijacked pointer */
  1011. /* should we use the first or second counter field
  1012. * of the hijacked pointer? */
  1013. int hi = (pageoff > PAGE_COUNTER_MASK);
  1014. csize = ((sector_t)1) << (CHUNK_BLOCK_SHIFT(bitmap) +
  1015. PAGE_COUNTER_SHIFT - 1);
  1016. *blocks = csize - (offset & (csize- 1));
  1017. return &((bitmap_counter_t *)
  1018. &bitmap->bp[page].map)[hi];
  1019. } else { /* page is allocated */
  1020. csize = ((sector_t)1) << (CHUNK_BLOCK_SHIFT(bitmap));
  1021. *blocks = csize - (offset & (csize- 1));
  1022. return (bitmap_counter_t *)
  1023. &(bitmap->bp[page].map[pageoff]);
  1024. }
  1025. }
  1026. int bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors, int behind)
  1027. {
  1028. if (!bitmap) return 0;
  1029. if (behind) {
  1030. atomic_inc(&bitmap->behind_writes);
  1031. PRINTK(KERN_DEBUG "inc write-behind count %d/%d\n",
  1032. atomic_read(&bitmap->behind_writes), bitmap->max_write_behind);
  1033. }
  1034. while (sectors) {
  1035. int blocks;
  1036. bitmap_counter_t *bmc;
  1037. spin_lock_irq(&bitmap->lock);
  1038. bmc = bitmap_get_counter(bitmap, offset, &blocks, 1);
  1039. if (!bmc) {
  1040. spin_unlock_irq(&bitmap->lock);
  1041. return 0;
  1042. }
  1043. if (unlikely((*bmc & COUNTER_MAX) == COUNTER_MAX)) {
  1044. DEFINE_WAIT(__wait);
  1045. /* note that it is safe to do the prepare_to_wait
  1046. * after the test as long as we do it before dropping
  1047. * the spinlock.
  1048. */
  1049. prepare_to_wait(&bitmap->overflow_wait, &__wait,
  1050. TASK_UNINTERRUPTIBLE);
  1051. spin_unlock_irq(&bitmap->lock);
  1052. blk_unplug(bitmap->mddev->queue);
  1053. schedule();
  1054. finish_wait(&bitmap->overflow_wait, &__wait);
  1055. continue;
  1056. }
  1057. switch(*bmc) {
  1058. case 0:
  1059. bitmap_file_set_bit(bitmap, offset);
  1060. bitmap_count_page(bitmap,offset, 1);
  1061. blk_plug_device(bitmap->mddev->queue);
  1062. /* fall through */
  1063. case 1:
  1064. *bmc = 2;
  1065. }
  1066. (*bmc)++;
  1067. spin_unlock_irq(&bitmap->lock);
  1068. offset += blocks;
  1069. if (sectors > blocks)
  1070. sectors -= blocks;
  1071. else sectors = 0;
  1072. }
  1073. bitmap->allclean = 0;
  1074. return 0;
  1075. }
  1076. void bitmap_endwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors,
  1077. int success, int behind)
  1078. {
  1079. if (!bitmap) return;
  1080. if (behind) {
  1081. atomic_dec(&bitmap->behind_writes);
  1082. PRINTK(KERN_DEBUG "dec write-behind count %d/%d\n",
  1083. atomic_read(&bitmap->behind_writes), bitmap->max_write_behind);
  1084. }
  1085. while (sectors) {
  1086. int blocks;
  1087. unsigned long flags;
  1088. bitmap_counter_t *bmc;
  1089. spin_lock_irqsave(&bitmap->lock, flags);
  1090. bmc = bitmap_get_counter(bitmap, offset, &blocks, 0);
  1091. if (!bmc) {
  1092. spin_unlock_irqrestore(&bitmap->lock, flags);
  1093. return;
  1094. }
  1095. if (!success && ! (*bmc & NEEDED_MASK))
  1096. *bmc |= NEEDED_MASK;
  1097. if ((*bmc & COUNTER_MAX) == COUNTER_MAX)
  1098. wake_up(&bitmap->overflow_wait);
  1099. (*bmc)--;
  1100. if (*bmc <= 2) {
  1101. set_page_attr(bitmap,
  1102. filemap_get_page(bitmap, offset >> CHUNK_BLOCK_SHIFT(bitmap)),
  1103. BITMAP_PAGE_CLEAN);
  1104. }
  1105. spin_unlock_irqrestore(&bitmap->lock, flags);
  1106. offset += blocks;
  1107. if (sectors > blocks)
  1108. sectors -= blocks;
  1109. else sectors = 0;
  1110. }
  1111. }
  1112. int bitmap_start_sync(struct bitmap *bitmap, sector_t offset, int *blocks,
  1113. int degraded)
  1114. {
  1115. bitmap_counter_t *bmc;
  1116. int rv;
  1117. if (bitmap == NULL) {/* FIXME or bitmap set as 'failed' */
  1118. *blocks = 1024;
  1119. return 1; /* always resync if no bitmap */
  1120. }
  1121. spin_lock_irq(&bitmap->lock);
  1122. bmc = bitmap_get_counter(bitmap, offset, blocks, 0);
  1123. rv = 0;
  1124. if (bmc) {
  1125. /* locked */
  1126. if (RESYNC(*bmc))
  1127. rv = 1;
  1128. else if (NEEDED(*bmc)) {
  1129. rv = 1;
  1130. if (!degraded) { /* don't set/clear bits if degraded */
  1131. *bmc |= RESYNC_MASK;
  1132. *bmc &= ~NEEDED_MASK;
  1133. }
  1134. }
  1135. }
  1136. spin_unlock_irq(&bitmap->lock);
  1137. bitmap->allclean = 0;
  1138. return rv;
  1139. }
  1140. void bitmap_end_sync(struct bitmap *bitmap, sector_t offset, int *blocks, int aborted)
  1141. {
  1142. bitmap_counter_t *bmc;
  1143. unsigned long flags;
  1144. /*
  1145. if (offset == 0) printk("bitmap_end_sync 0 (%d)\n", aborted);
  1146. */ if (bitmap == NULL) {
  1147. *blocks = 1024;
  1148. return;
  1149. }
  1150. spin_lock_irqsave(&bitmap->lock, flags);
  1151. bmc = bitmap_get_counter(bitmap, offset, blocks, 0);
  1152. if (bmc == NULL)
  1153. goto unlock;
  1154. /* locked */
  1155. /*
  1156. if (offset == 0) printk("bitmap_end sync found 0x%x, blocks %d\n", *bmc, *blocks);
  1157. */
  1158. if (RESYNC(*bmc)) {
  1159. *bmc &= ~RESYNC_MASK;
  1160. if (!NEEDED(*bmc) && aborted)
  1161. *bmc |= NEEDED_MASK;
  1162. else {
  1163. if (*bmc <= 2) {
  1164. set_page_attr(bitmap,
  1165. filemap_get_page(bitmap, offset >> CHUNK_BLOCK_SHIFT(bitmap)),
  1166. BITMAP_PAGE_CLEAN);
  1167. }
  1168. }
  1169. }
  1170. unlock:
  1171. spin_unlock_irqrestore(&bitmap->lock, flags);
  1172. bitmap->allclean = 0;
  1173. }
  1174. void bitmap_close_sync(struct bitmap *bitmap)
  1175. {
  1176. /* Sync has finished, and any bitmap chunks that weren't synced
  1177. * properly have been aborted. It remains to us to clear the
  1178. * RESYNC bit wherever it is still on
  1179. */
  1180. sector_t sector = 0;
  1181. int blocks;
  1182. if (!bitmap)
  1183. return;
  1184. while (sector < bitmap->mddev->resync_max_sectors) {
  1185. bitmap_end_sync(bitmap, sector, &blocks, 0);
  1186. sector += blocks;
  1187. }
  1188. }
  1189. void bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector)
  1190. {
  1191. sector_t s = 0;
  1192. int blocks;
  1193. if (!bitmap)
  1194. return;
  1195. if (sector == 0) {
  1196. bitmap->last_end_sync = jiffies;
  1197. return;
  1198. }
  1199. if (time_before(jiffies, (bitmap->last_end_sync
  1200. + bitmap->daemon_sleep * HZ)))
  1201. return;
  1202. wait_event(bitmap->mddev->recovery_wait,
  1203. atomic_read(&bitmap->mddev->recovery_active) == 0);
  1204. sector &= ~((1ULL << CHUNK_BLOCK_SHIFT(bitmap)) - 1);
  1205. s = 0;
  1206. while (s < sector && s < bitmap->mddev->resync_max_sectors) {
  1207. bitmap_end_sync(bitmap, s, &blocks, 0);
  1208. s += blocks;
  1209. }
  1210. bitmap->last_end_sync = jiffies;
  1211. }
  1212. static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed)
  1213. {
  1214. /* For each chunk covered by any of these sectors, set the
  1215. * counter to 1 and set resync_needed. They should all
  1216. * be 0 at this point
  1217. */
  1218. int secs;
  1219. bitmap_counter_t *bmc;
  1220. spin_lock_irq(&bitmap->lock);
  1221. bmc = bitmap_get_counter(bitmap, offset, &secs, 1);
  1222. if (!bmc) {
  1223. spin_unlock_irq(&bitmap->lock);
  1224. return;
  1225. }
  1226. if (! *bmc) {
  1227. struct page *page;
  1228. *bmc = 1 | (needed?NEEDED_MASK:0);
  1229. bitmap_count_page(bitmap, offset, 1);
  1230. page = filemap_get_page(bitmap, offset >> CHUNK_BLOCK_SHIFT(bitmap));
  1231. set_page_attr(bitmap, page, BITMAP_PAGE_CLEAN);
  1232. }
  1233. spin_unlock_irq(&bitmap->lock);
  1234. bitmap->allclean = 0;
  1235. }
  1236. /* dirty the memory and file bits for bitmap chunks "s" to "e" */
  1237. void bitmap_dirty_bits(struct bitmap *bitmap, unsigned long s, unsigned long e)
  1238. {
  1239. unsigned long chunk;
  1240. for (chunk = s; chunk <= e; chunk++) {
  1241. sector_t sec = chunk << CHUNK_BLOCK_SHIFT(bitmap);
  1242. bitmap_set_memory_bits(bitmap, sec, 1);
  1243. bitmap_file_set_bit(bitmap, sec);
  1244. }
  1245. }
  1246. /*
  1247. * flush out any pending updates
  1248. */
  1249. void bitmap_flush(mddev_t *mddev)
  1250. {
  1251. struct bitmap *bitmap = mddev->bitmap;
  1252. int sleep;
  1253. if (!bitmap) /* there was no bitmap */
  1254. return;
  1255. /* run the daemon_work three time to ensure everything is flushed
  1256. * that can be
  1257. */
  1258. sleep = bitmap->daemon_sleep;
  1259. bitmap->daemon_sleep = 0;
  1260. bitmap_daemon_work(bitmap);
  1261. bitmap_daemon_work(bitmap);
  1262. bitmap_daemon_work(bitmap);
  1263. bitmap->daemon_sleep = sleep;
  1264. bitmap_update_sb(bitmap);
  1265. }
  1266. /*
  1267. * free memory that was allocated
  1268. */
  1269. static void bitmap_free(struct bitmap *bitmap)
  1270. {
  1271. unsigned long k, pages;
  1272. struct bitmap_page *bp;
  1273. if (!bitmap) /* there was no bitmap */
  1274. return;
  1275. /* release the bitmap file and kill the daemon */
  1276. bitmap_file_put(bitmap);
  1277. bp = bitmap->bp;
  1278. pages = bitmap->pages;
  1279. /* free all allocated memory */
  1280. if (bp) /* deallocate the page memory */
  1281. for (k = 0; k < pages; k++)
  1282. if (bp[k].map && !bp[k].hijacked)
  1283. kfree(bp[k].map);
  1284. kfree(bp);
  1285. kfree(bitmap);
  1286. }
  1287. void bitmap_destroy(mddev_t *mddev)
  1288. {
  1289. struct bitmap *bitmap = mddev->bitmap;
  1290. if (!bitmap) /* there was no bitmap */
  1291. return;
  1292. mddev->bitmap = NULL; /* disconnect from the md device */
  1293. if (mddev->thread)
  1294. mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
  1295. bitmap_free(bitmap);
  1296. }
  1297. /*
  1298. * initialize the bitmap structure
  1299. * if this returns an error, bitmap_destroy must be called to do clean up
  1300. */
  1301. int bitmap_create(mddev_t *mddev)
  1302. {
  1303. struct bitmap *bitmap;
  1304. unsigned long blocks = mddev->resync_max_sectors;
  1305. unsigned long chunks;
  1306. unsigned long pages;
  1307. struct file *file = mddev->bitmap_file;
  1308. int err;
  1309. sector_t start;
  1310. BUILD_BUG_ON(sizeof(bitmap_super_t) != 256);
  1311. if (!file && !mddev->bitmap_offset) /* bitmap disabled, nothing to do */
  1312. return 0;
  1313. BUG_ON(file && mddev->bitmap_offset);
  1314. bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL);
  1315. if (!bitmap)
  1316. return -ENOMEM;
  1317. spin_lock_init(&bitmap->lock);
  1318. atomic_set(&bitmap->pending_writes, 0);
  1319. init_waitqueue_head(&bitmap->write_wait);
  1320. init_waitqueue_head(&bitmap->overflow_wait);
  1321. bitmap->mddev = mddev;
  1322. bitmap->file = file;
  1323. bitmap->offset = mddev->bitmap_offset;
  1324. if (file) {
  1325. get_file(file);
  1326. do_sync_mapping_range(file->f_mapping, 0, LLONG_MAX,
  1327. SYNC_FILE_RANGE_WAIT_BEFORE |
  1328. SYNC_FILE_RANGE_WRITE |
  1329. SYNC_FILE_RANGE_WAIT_AFTER);
  1330. }
  1331. /* read superblock from bitmap file (this sets bitmap->chunksize) */
  1332. err = bitmap_read_sb(bitmap);
  1333. if (err)
  1334. goto error;
  1335. bitmap->chunkshift = ffz(~bitmap->chunksize);
  1336. /* now that chunksize and chunkshift are set, we can use these macros */
  1337. chunks = (blocks + CHUNK_BLOCK_RATIO(bitmap) - 1) /
  1338. CHUNK_BLOCK_RATIO(bitmap);
  1339. pages = (chunks + PAGE_COUNTER_RATIO - 1) / PAGE_COUNTER_RATIO;
  1340. BUG_ON(!pages);
  1341. bitmap->chunks = chunks;
  1342. bitmap->pages = pages;
  1343. bitmap->missing_pages = pages;
  1344. bitmap->counter_bits = COUNTER_BITS;
  1345. bitmap->syncchunk = ~0UL;
  1346. #ifdef INJECT_FATAL_FAULT_1
  1347. bitmap->bp = NULL;
  1348. #else
  1349. bitmap->bp = kzalloc(pages * sizeof(*bitmap->bp), GFP_KERNEL);
  1350. #endif
  1351. err = -ENOMEM;
  1352. if (!bitmap->bp)
  1353. goto error;
  1354. /* now that we have some pages available, initialize the in-memory
  1355. * bitmap from the on-disk bitmap */
  1356. start = 0;
  1357. if (mddev->degraded == 0
  1358. || bitmap->events_cleared == mddev->events)
  1359. /* no need to keep dirty bits to optimise a re-add of a missing device */
  1360. start = mddev->recovery_cp;
  1361. err = bitmap_init_from_disk(bitmap, start);
  1362. if (err)
  1363. goto error;
  1364. printk(KERN_INFO "created bitmap (%lu pages) for device %s\n",
  1365. pages, bmname(bitmap));
  1366. mddev->bitmap = bitmap;
  1367. mddev->thread->timeout = bitmap->daemon_sleep * HZ;
  1368. bitmap_update_sb(bitmap);
  1369. return (bitmap->flags & BITMAP_WRITE_ERROR) ? -EIO : 0;
  1370. error:
  1371. bitmap_free(bitmap);
  1372. return err;
  1373. }
  1374. /* the bitmap API -- for raid personalities */
  1375. EXPORT_SYMBOL(bitmap_startwrite);
  1376. EXPORT_SYMBOL(bitmap_endwrite);
  1377. EXPORT_SYMBOL(bitmap_start_sync);
  1378. EXPORT_SYMBOL(bitmap_end_sync);
  1379. EXPORT_SYMBOL(bitmap_unplug);
  1380. EXPORT_SYMBOL(bitmap_close_sync);
  1381. EXPORT_SYMBOL(bitmap_cond_end_sync);