bitmap.c 39 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508
  1. /*
  2. * bitmap.c two-level bitmap (C) Peter T. Breuer (ptb@ot.uc3m.es) 2003
  3. *
  4. * bitmap_create - sets up the bitmap structure
  5. * bitmap_destroy - destroys the bitmap structure
  6. *
  7. * additions, Copyright (C) 2003-2004, Paul Clements, SteelEye Technology, Inc.:
  8. * - added disk storage for bitmap
  9. * - changes to allow various bitmap chunk sizes
  10. */
  11. /*
  12. * Still to do:
  13. *
  14. * flush after percent set rather than just time based. (maybe both).
  15. * wait if count gets too high, wake when it drops to half.
  16. */
  17. #include <linux/module.h>
  18. #include <linux/errno.h>
  19. #include <linux/slab.h>
  20. #include <linux/init.h>
  21. #include <linux/timer.h>
  22. #include <linux/sched.h>
  23. #include <linux/list.h>
  24. #include <linux/file.h>
  25. #include <linux/mount.h>
  26. #include <linux/buffer_head.h>
  27. #include <linux/raid/md.h>
  28. #include <linux/raid/bitmap.h>
  29. /* debug macros */
  30. #define DEBUG 0
  31. #if DEBUG
  32. /* these are for debugging purposes only! */
  33. /* define one and only one of these */
  34. #define INJECT_FAULTS_1 0 /* cause bitmap_alloc_page to fail always */
  35. #define INJECT_FAULTS_2 0 /* cause bitmap file to be kicked when first bit set*/
  36. #define INJECT_FAULTS_3 0 /* treat bitmap file as kicked at init time */
  37. #define INJECT_FAULTS_4 0 /* undef */
  38. #define INJECT_FAULTS_5 0 /* undef */
  39. #define INJECT_FAULTS_6 0
  40. /* if these are defined, the driver will fail! debug only */
  41. #define INJECT_FATAL_FAULT_1 0 /* fail kmalloc, causing bitmap_create to fail */
  42. #define INJECT_FATAL_FAULT_2 0 /* undef */
  43. #define INJECT_FATAL_FAULT_3 0 /* undef */
  44. #endif
  45. //#define DPRINTK PRINTK /* set this NULL to avoid verbose debug output */
  46. #define DPRINTK(x...) do { } while(0)
  47. #ifndef PRINTK
  48. # if DEBUG > 0
  49. # define PRINTK(x...) printk(KERN_DEBUG x)
  50. # else
  51. # define PRINTK(x...)
  52. # endif
  53. #endif
  54. static inline char * bmname(struct bitmap *bitmap)
  55. {
  56. return bitmap->mddev ? mdname(bitmap->mddev) : "mdX";
  57. }
  58. /*
  59. * just a placeholder - calls kmalloc for bitmap pages
  60. */
  61. static unsigned char *bitmap_alloc_page(struct bitmap *bitmap)
  62. {
  63. unsigned char *page;
  64. #ifdef INJECT_FAULTS_1
  65. page = NULL;
  66. #else
  67. page = kmalloc(PAGE_SIZE, GFP_NOIO);
  68. #endif
  69. if (!page)
  70. printk("%s: bitmap_alloc_page FAILED\n", bmname(bitmap));
  71. else
  72. PRINTK("%s: bitmap_alloc_page: allocated page at %p\n",
  73. bmname(bitmap), page);
  74. return page;
  75. }
  76. /*
  77. * for now just a placeholder -- just calls kfree for bitmap pages
  78. */
  79. static void bitmap_free_page(struct bitmap *bitmap, unsigned char *page)
  80. {
  81. PRINTK("%s: bitmap_free_page: free page %p\n", bmname(bitmap), page);
  82. kfree(page);
  83. }
  84. /*
  85. * check a page and, if necessary, allocate it (or hijack it if the alloc fails)
  86. *
  87. * 1) check to see if this page is allocated, if it's not then try to alloc
  88. * 2) if the alloc fails, set the page's hijacked flag so we'll use the
  89. * page pointer directly as a counter
  90. *
  91. * if we find our page, we increment the page's refcount so that it stays
  92. * allocated while we're using it
  93. */
  94. static int bitmap_checkpage(struct bitmap *bitmap, unsigned long page, int create)
  95. {
  96. unsigned char *mappage;
  97. if (page >= bitmap->pages) {
  98. printk(KERN_ALERT
  99. "%s: invalid bitmap page request: %lu (> %lu)\n",
  100. bmname(bitmap), page, bitmap->pages-1);
  101. return -EINVAL;
  102. }
  103. if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */
  104. return 0;
  105. if (bitmap->bp[page].map) /* page is already allocated, just return */
  106. return 0;
  107. if (!create)
  108. return -ENOENT;
  109. spin_unlock_irq(&bitmap->lock);
  110. /* this page has not been allocated yet */
  111. if ((mappage = bitmap_alloc_page(bitmap)) == NULL) {
  112. PRINTK("%s: bitmap map page allocation failed, hijacking\n",
  113. bmname(bitmap));
  114. /* failed - set the hijacked flag so that we can use the
  115. * pointer as a counter */
  116. spin_lock_irq(&bitmap->lock);
  117. if (!bitmap->bp[page].map)
  118. bitmap->bp[page].hijacked = 1;
  119. goto out;
  120. }
  121. /* got a page */
  122. spin_lock_irq(&bitmap->lock);
  123. /* recheck the page */
  124. if (bitmap->bp[page].map || bitmap->bp[page].hijacked) {
  125. /* somebody beat us to getting the page */
  126. bitmap_free_page(bitmap, mappage);
  127. return 0;
  128. }
  129. /* no page was in place and we have one, so install it */
  130. memset(mappage, 0, PAGE_SIZE);
  131. bitmap->bp[page].map = mappage;
  132. bitmap->missing_pages--;
  133. out:
  134. return 0;
  135. }
  136. /* if page is completely empty, put it back on the free list, or dealloc it */
  137. /* if page was hijacked, unmark the flag so it might get alloced next time */
  138. /* Note: lock should be held when calling this */
  139. static void bitmap_checkfree(struct bitmap *bitmap, unsigned long page)
  140. {
  141. char *ptr;
  142. if (bitmap->bp[page].count) /* page is still busy */
  143. return;
  144. /* page is no longer in use, it can be released */
  145. if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */
  146. bitmap->bp[page].hijacked = 0;
  147. bitmap->bp[page].map = NULL;
  148. return;
  149. }
  150. /* normal case, free the page */
  151. #if 0
  152. /* actually ... let's not. We will probably need the page again exactly when
  153. * memory is tight and we are flusing to disk
  154. */
  155. return;
  156. #else
  157. ptr = bitmap->bp[page].map;
  158. bitmap->bp[page].map = NULL;
  159. bitmap->missing_pages++;
  160. bitmap_free_page(bitmap, ptr);
  161. return;
  162. #endif
  163. }
  164. /*
  165. * bitmap file handling - read and write the bitmap file and its superblock
  166. */
  167. /* copy the pathname of a file to a buffer */
  168. char *file_path(struct file *file, char *buf, int count)
  169. {
  170. struct dentry *d;
  171. struct vfsmount *v;
  172. if (!buf)
  173. return NULL;
  174. d = file->f_path.dentry;
  175. v = file->f_path.mnt;
  176. buf = d_path(d, v, buf, count);
  177. return IS_ERR(buf) ? NULL : buf;
  178. }
  179. /*
  180. * basic page I/O operations
  181. */
  182. /* IO operations when bitmap is stored near all superblocks */
  183. static struct page *read_sb_page(mddev_t *mddev, long offset, unsigned long index)
  184. {
  185. /* choose a good rdev and read the page from there */
  186. mdk_rdev_t *rdev;
  187. struct list_head *tmp;
  188. struct page *page = alloc_page(GFP_KERNEL);
  189. sector_t target;
  190. if (!page)
  191. return ERR_PTR(-ENOMEM);
  192. ITERATE_RDEV(mddev, rdev, tmp) {
  193. if (! test_bit(In_sync, &rdev->flags)
  194. || test_bit(Faulty, &rdev->flags))
  195. continue;
  196. target = (rdev->sb_offset << 1) + offset + index * (PAGE_SIZE/512);
  197. if (sync_page_io(rdev->bdev, target, PAGE_SIZE, page, READ)) {
  198. page->index = index;
  199. attach_page_buffers(page, NULL); /* so that free_buffer will
  200. * quietly no-op */
  201. return page;
  202. }
  203. }
  204. return ERR_PTR(-EIO);
  205. }
  206. static int write_sb_page(mddev_t *mddev, long offset, struct page *page, int wait)
  207. {
  208. mdk_rdev_t *rdev;
  209. struct list_head *tmp;
  210. ITERATE_RDEV(mddev, rdev, tmp)
  211. if (test_bit(In_sync, &rdev->flags)
  212. && !test_bit(Faulty, &rdev->flags))
  213. md_super_write(mddev, rdev,
  214. (rdev->sb_offset<<1) + offset
  215. + page->index * (PAGE_SIZE/512),
  216. PAGE_SIZE,
  217. page);
  218. if (wait)
  219. md_super_wait(mddev);
  220. return 0;
  221. }
  222. /*
  223. * write out a page to a file
  224. */
  225. static int write_page(struct bitmap *bitmap, struct page *page, int wait)
  226. {
  227. struct buffer_head *bh;
  228. if (bitmap->file == NULL)
  229. return write_sb_page(bitmap->mddev, bitmap->offset, page, wait);
  230. bh = page_buffers(page);
  231. while (bh && bh->b_blocknr) {
  232. atomic_inc(&bitmap->pending_writes);
  233. set_buffer_locked(bh);
  234. set_buffer_mapped(bh);
  235. submit_bh(WRITE, bh);
  236. bh = bh->b_this_page;
  237. }
  238. if (wait) {
  239. wait_event(bitmap->write_wait,
  240. atomic_read(&bitmap->pending_writes)==0);
  241. return (bitmap->flags & BITMAP_WRITE_ERROR) ? -EIO : 0;
  242. }
  243. return 0;
  244. }
  245. static void end_bitmap_write(struct buffer_head *bh, int uptodate)
  246. {
  247. struct bitmap *bitmap = bh->b_private;
  248. unsigned long flags;
  249. if (!uptodate) {
  250. spin_lock_irqsave(&bitmap->lock, flags);
  251. bitmap->flags |= BITMAP_WRITE_ERROR;
  252. spin_unlock_irqrestore(&bitmap->lock, flags);
  253. }
  254. if (atomic_dec_and_test(&bitmap->pending_writes))
  255. wake_up(&bitmap->write_wait);
  256. }
  257. /* copied from buffer.c */
  258. static void
  259. __clear_page_buffers(struct page *page)
  260. {
  261. ClearPagePrivate(page);
  262. set_page_private(page, 0);
  263. page_cache_release(page);
  264. }
  265. static void free_buffers(struct page *page)
  266. {
  267. struct buffer_head *bh = page_buffers(page);
  268. while (bh) {
  269. struct buffer_head *next = bh->b_this_page;
  270. free_buffer_head(bh);
  271. bh = next;
  272. }
  273. __clear_page_buffers(page);
  274. put_page(page);
  275. }
  276. /* read a page from a file.
  277. * We both read the page, and attach buffers to the page to record the
  278. * address of each block (using bmap). These addresses will be used
  279. * to write the block later, completely bypassing the filesystem.
  280. * This usage is similar to how swap files are handled, and allows us
  281. * to write to a file with no concerns of memory allocation failing.
  282. */
  283. static struct page *read_page(struct file *file, unsigned long index,
  284. struct bitmap *bitmap,
  285. unsigned long count)
  286. {
  287. struct page *page = NULL;
  288. struct inode *inode = file->f_path.dentry->d_inode;
  289. struct buffer_head *bh;
  290. sector_t block;
  291. PRINTK("read bitmap file (%dB @ %Lu)\n", (int)PAGE_SIZE,
  292. (unsigned long long)index << PAGE_SHIFT);
  293. page = alloc_page(GFP_KERNEL);
  294. if (!page)
  295. page = ERR_PTR(-ENOMEM);
  296. if (IS_ERR(page))
  297. goto out;
  298. bh = alloc_page_buffers(page, 1<<inode->i_blkbits, 0);
  299. if (!bh) {
  300. put_page(page);
  301. page = ERR_PTR(-ENOMEM);
  302. goto out;
  303. }
  304. attach_page_buffers(page, bh);
  305. block = index << (PAGE_SHIFT - inode->i_blkbits);
  306. while (bh) {
  307. if (count == 0)
  308. bh->b_blocknr = 0;
  309. else {
  310. bh->b_blocknr = bmap(inode, block);
  311. if (bh->b_blocknr == 0) {
  312. /* Cannot use this file! */
  313. free_buffers(page);
  314. page = ERR_PTR(-EINVAL);
  315. goto out;
  316. }
  317. bh->b_bdev = inode->i_sb->s_bdev;
  318. if (count < (1<<inode->i_blkbits))
  319. count = 0;
  320. else
  321. count -= (1<<inode->i_blkbits);
  322. bh->b_end_io = end_bitmap_write;
  323. bh->b_private = bitmap;
  324. atomic_inc(&bitmap->pending_writes);
  325. set_buffer_locked(bh);
  326. set_buffer_mapped(bh);
  327. submit_bh(READ, bh);
  328. }
  329. block++;
  330. bh = bh->b_this_page;
  331. }
  332. page->index = index;
  333. wait_event(bitmap->write_wait,
  334. atomic_read(&bitmap->pending_writes)==0);
  335. if (bitmap->flags & BITMAP_WRITE_ERROR) {
  336. free_buffers(page);
  337. page = ERR_PTR(-EIO);
  338. }
  339. out:
  340. if (IS_ERR(page))
  341. printk(KERN_ALERT "md: bitmap read error: (%dB @ %Lu): %ld\n",
  342. (int)PAGE_SIZE,
  343. (unsigned long long)index << PAGE_SHIFT,
  344. PTR_ERR(page));
  345. return page;
  346. }
  347. /*
  348. * bitmap file superblock operations
  349. */
  350. /* update the event counter and sync the superblock to disk */
  351. int bitmap_update_sb(struct bitmap *bitmap)
  352. {
  353. bitmap_super_t *sb;
  354. unsigned long flags;
  355. if (!bitmap || !bitmap->mddev) /* no bitmap for this array */
  356. return 0;
  357. spin_lock_irqsave(&bitmap->lock, flags);
  358. if (!bitmap->sb_page) { /* no superblock */
  359. spin_unlock_irqrestore(&bitmap->lock, flags);
  360. return 0;
  361. }
  362. spin_unlock_irqrestore(&bitmap->lock, flags);
  363. sb = (bitmap_super_t *)kmap_atomic(bitmap->sb_page, KM_USER0);
  364. sb->events = cpu_to_le64(bitmap->mddev->events);
  365. if (!bitmap->mddev->degraded)
  366. sb->events_cleared = cpu_to_le64(bitmap->mddev->events);
  367. kunmap_atomic(sb, KM_USER0);
  368. return write_page(bitmap, bitmap->sb_page, 1);
  369. }
  370. /* print out the bitmap file superblock */
  371. void bitmap_print_sb(struct bitmap *bitmap)
  372. {
  373. bitmap_super_t *sb;
  374. if (!bitmap || !bitmap->sb_page)
  375. return;
  376. sb = (bitmap_super_t *)kmap_atomic(bitmap->sb_page, KM_USER0);
  377. printk(KERN_DEBUG "%s: bitmap file superblock:\n", bmname(bitmap));
  378. printk(KERN_DEBUG " magic: %08x\n", le32_to_cpu(sb->magic));
  379. printk(KERN_DEBUG " version: %d\n", le32_to_cpu(sb->version));
  380. printk(KERN_DEBUG " uuid: %08x.%08x.%08x.%08x\n",
  381. *(__u32 *)(sb->uuid+0),
  382. *(__u32 *)(sb->uuid+4),
  383. *(__u32 *)(sb->uuid+8),
  384. *(__u32 *)(sb->uuid+12));
  385. printk(KERN_DEBUG " events: %llu\n",
  386. (unsigned long long) le64_to_cpu(sb->events));
  387. printk(KERN_DEBUG "events cleared: %llu\n",
  388. (unsigned long long) le64_to_cpu(sb->events_cleared));
  389. printk(KERN_DEBUG " state: %08x\n", le32_to_cpu(sb->state));
  390. printk(KERN_DEBUG " chunksize: %d B\n", le32_to_cpu(sb->chunksize));
  391. printk(KERN_DEBUG " daemon sleep: %ds\n", le32_to_cpu(sb->daemon_sleep));
  392. printk(KERN_DEBUG " sync size: %llu KB\n",
  393. (unsigned long long)le64_to_cpu(sb->sync_size)/2);
  394. printk(KERN_DEBUG "max write behind: %d\n", le32_to_cpu(sb->write_behind));
  395. kunmap_atomic(sb, KM_USER0);
  396. }
  397. /* read the superblock from the bitmap file and initialize some bitmap fields */
  398. static int bitmap_read_sb(struct bitmap *bitmap)
  399. {
  400. char *reason = NULL;
  401. bitmap_super_t *sb;
  402. unsigned long chunksize, daemon_sleep, write_behind;
  403. unsigned long long events;
  404. int err = -EINVAL;
  405. /* page 0 is the superblock, read it... */
  406. if (bitmap->file) {
  407. loff_t isize = i_size_read(bitmap->file->f_mapping->host);
  408. int bytes = isize > PAGE_SIZE ? PAGE_SIZE : isize;
  409. bitmap->sb_page = read_page(bitmap->file, 0, bitmap, bytes);
  410. } else {
  411. bitmap->sb_page = read_sb_page(bitmap->mddev, bitmap->offset, 0);
  412. }
  413. if (IS_ERR(bitmap->sb_page)) {
  414. err = PTR_ERR(bitmap->sb_page);
  415. bitmap->sb_page = NULL;
  416. return err;
  417. }
  418. sb = (bitmap_super_t *)kmap_atomic(bitmap->sb_page, KM_USER0);
  419. chunksize = le32_to_cpu(sb->chunksize);
  420. daemon_sleep = le32_to_cpu(sb->daemon_sleep);
  421. write_behind = le32_to_cpu(sb->write_behind);
  422. /* verify that the bitmap-specific fields are valid */
  423. if (sb->magic != cpu_to_le32(BITMAP_MAGIC))
  424. reason = "bad magic";
  425. else if (le32_to_cpu(sb->version) < BITMAP_MAJOR_LO ||
  426. le32_to_cpu(sb->version) > BITMAP_MAJOR_HI)
  427. reason = "unrecognized superblock version";
  428. else if (chunksize < PAGE_SIZE)
  429. reason = "bitmap chunksize too small";
  430. else if ((1 << ffz(~chunksize)) != chunksize)
  431. reason = "bitmap chunksize not a power of 2";
  432. else if (daemon_sleep < 1 || daemon_sleep > MAX_SCHEDULE_TIMEOUT / HZ)
  433. reason = "daemon sleep period out of range";
  434. else if (write_behind > COUNTER_MAX)
  435. reason = "write-behind limit out of range (0 - 16383)";
  436. if (reason) {
  437. printk(KERN_INFO "%s: invalid bitmap file superblock: %s\n",
  438. bmname(bitmap), reason);
  439. goto out;
  440. }
  441. /* keep the array size field of the bitmap superblock up to date */
  442. sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
  443. if (!bitmap->mddev->persistent)
  444. goto success;
  445. /*
  446. * if we have a persistent array superblock, compare the
  447. * bitmap's UUID and event counter to the mddev's
  448. */
  449. if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) {
  450. printk(KERN_INFO "%s: bitmap superblock UUID mismatch\n",
  451. bmname(bitmap));
  452. goto out;
  453. }
  454. events = le64_to_cpu(sb->events);
  455. if (events < bitmap->mddev->events) {
  456. printk(KERN_INFO "%s: bitmap file is out of date (%llu < %llu) "
  457. "-- forcing full recovery\n", bmname(bitmap), events,
  458. (unsigned long long) bitmap->mddev->events);
  459. sb->state |= cpu_to_le32(BITMAP_STALE);
  460. }
  461. success:
  462. /* assign fields using values from superblock */
  463. bitmap->chunksize = chunksize;
  464. bitmap->daemon_sleep = daemon_sleep;
  465. bitmap->daemon_lastrun = jiffies;
  466. bitmap->max_write_behind = write_behind;
  467. bitmap->flags |= le32_to_cpu(sb->state);
  468. if (le32_to_cpu(sb->version) == BITMAP_MAJOR_HOSTENDIAN)
  469. bitmap->flags |= BITMAP_HOSTENDIAN;
  470. bitmap->events_cleared = le64_to_cpu(sb->events_cleared);
  471. if (sb->state & cpu_to_le32(BITMAP_STALE))
  472. bitmap->events_cleared = bitmap->mddev->events;
  473. err = 0;
  474. out:
  475. kunmap_atomic(sb, KM_USER0);
  476. if (err)
  477. bitmap_print_sb(bitmap);
  478. return err;
  479. }
  480. enum bitmap_mask_op {
  481. MASK_SET,
  482. MASK_UNSET
  483. };
  484. /* record the state of the bitmap in the superblock */
  485. static void bitmap_mask_state(struct bitmap *bitmap, enum bitmap_state bits,
  486. enum bitmap_mask_op op)
  487. {
  488. bitmap_super_t *sb;
  489. unsigned long flags;
  490. spin_lock_irqsave(&bitmap->lock, flags);
  491. if (!bitmap->sb_page) { /* can't set the state */
  492. spin_unlock_irqrestore(&bitmap->lock, flags);
  493. return;
  494. }
  495. spin_unlock_irqrestore(&bitmap->lock, flags);
  496. sb = (bitmap_super_t *)kmap_atomic(bitmap->sb_page, KM_USER0);
  497. switch (op) {
  498. case MASK_SET: sb->state |= cpu_to_le32(bits);
  499. break;
  500. case MASK_UNSET: sb->state &= cpu_to_le32(~bits);
  501. break;
  502. default: BUG();
  503. }
  504. kunmap_atomic(sb, KM_USER0);
  505. }
  506. /*
  507. * general bitmap file operations
  508. */
  509. /* calculate the index of the page that contains this bit */
  510. static inline unsigned long file_page_index(unsigned long chunk)
  511. {
  512. return CHUNK_BIT_OFFSET(chunk) >> PAGE_BIT_SHIFT;
  513. }
  514. /* calculate the (bit) offset of this bit within a page */
  515. static inline unsigned long file_page_offset(unsigned long chunk)
  516. {
  517. return CHUNK_BIT_OFFSET(chunk) & (PAGE_BITS - 1);
  518. }
  519. /*
  520. * return a pointer to the page in the filemap that contains the given bit
  521. *
  522. * this lookup is complicated by the fact that the bitmap sb might be exactly
  523. * 1 page (e.g., x86) or less than 1 page -- so the bitmap might start on page
  524. * 0 or page 1
  525. */
  526. static inline struct page *filemap_get_page(struct bitmap *bitmap,
  527. unsigned long chunk)
  528. {
  529. if (file_page_index(chunk) >= bitmap->file_pages) return NULL;
  530. return bitmap->filemap[file_page_index(chunk) - file_page_index(0)];
  531. }
  532. static void bitmap_file_unmap(struct bitmap *bitmap)
  533. {
  534. struct page **map, *sb_page;
  535. unsigned long *attr;
  536. int pages;
  537. unsigned long flags;
  538. spin_lock_irqsave(&bitmap->lock, flags);
  539. map = bitmap->filemap;
  540. bitmap->filemap = NULL;
  541. attr = bitmap->filemap_attr;
  542. bitmap->filemap_attr = NULL;
  543. pages = bitmap->file_pages;
  544. bitmap->file_pages = 0;
  545. sb_page = bitmap->sb_page;
  546. bitmap->sb_page = NULL;
  547. spin_unlock_irqrestore(&bitmap->lock, flags);
  548. while (pages--)
  549. if (map[pages]->index != 0) /* 0 is sb_page, release it below */
  550. free_buffers(map[pages]);
  551. kfree(map);
  552. kfree(attr);
  553. if (sb_page)
  554. free_buffers(sb_page);
  555. }
  556. static void bitmap_file_put(struct bitmap *bitmap)
  557. {
  558. struct file *file;
  559. unsigned long flags;
  560. spin_lock_irqsave(&bitmap->lock, flags);
  561. file = bitmap->file;
  562. bitmap->file = NULL;
  563. spin_unlock_irqrestore(&bitmap->lock, flags);
  564. if (file)
  565. wait_event(bitmap->write_wait,
  566. atomic_read(&bitmap->pending_writes)==0);
  567. bitmap_file_unmap(bitmap);
  568. if (file) {
  569. struct inode *inode = file->f_path.dentry->d_inode;
  570. invalidate_inode_pages(inode->i_mapping);
  571. fput(file);
  572. }
  573. }
  574. /*
  575. * bitmap_file_kick - if an error occurs while manipulating the bitmap file
  576. * then it is no longer reliable, so we stop using it and we mark the file
  577. * as failed in the superblock
  578. */
  579. static void bitmap_file_kick(struct bitmap *bitmap)
  580. {
  581. char *path, *ptr = NULL;
  582. bitmap_mask_state(bitmap, BITMAP_STALE, MASK_SET);
  583. bitmap_update_sb(bitmap);
  584. if (bitmap->file) {
  585. path = kmalloc(PAGE_SIZE, GFP_KERNEL);
  586. if (path)
  587. ptr = file_path(bitmap->file, path, PAGE_SIZE);
  588. printk(KERN_ALERT "%s: kicking failed bitmap file %s from array!\n",
  589. bmname(bitmap), ptr ? ptr : "");
  590. kfree(path);
  591. }
  592. bitmap_file_put(bitmap);
  593. return;
  594. }
  595. enum bitmap_page_attr {
  596. BITMAP_PAGE_DIRTY = 0, // there are set bits that need to be synced
  597. BITMAP_PAGE_CLEAN = 1, // there are bits that might need to be cleared
  598. BITMAP_PAGE_NEEDWRITE=2, // there are cleared bits that need to be synced
  599. };
  600. static inline void set_page_attr(struct bitmap *bitmap, struct page *page,
  601. enum bitmap_page_attr attr)
  602. {
  603. __set_bit((page->index<<2) + attr, bitmap->filemap_attr);
  604. }
  605. static inline void clear_page_attr(struct bitmap *bitmap, struct page *page,
  606. enum bitmap_page_attr attr)
  607. {
  608. __clear_bit((page->index<<2) + attr, bitmap->filemap_attr);
  609. }
  610. static inline unsigned long test_page_attr(struct bitmap *bitmap, struct page *page,
  611. enum bitmap_page_attr attr)
  612. {
  613. return test_bit((page->index<<2) + attr, bitmap->filemap_attr);
  614. }
  615. /*
  616. * bitmap_file_set_bit -- called before performing a write to the md device
  617. * to set (and eventually sync) a particular bit in the bitmap file
  618. *
  619. * we set the bit immediately, then we record the page number so that
  620. * when an unplug occurs, we can flush the dirty pages out to disk
  621. */
  622. static void bitmap_file_set_bit(struct bitmap *bitmap, sector_t block)
  623. {
  624. unsigned long bit;
  625. struct page *page;
  626. void *kaddr;
  627. unsigned long chunk = block >> CHUNK_BLOCK_SHIFT(bitmap);
  628. if (!bitmap->filemap) {
  629. return;
  630. }
  631. page = filemap_get_page(bitmap, chunk);
  632. if (!page) return;
  633. bit = file_page_offset(chunk);
  634. /* set the bit */
  635. kaddr = kmap_atomic(page, KM_USER0);
  636. if (bitmap->flags & BITMAP_HOSTENDIAN)
  637. set_bit(bit, kaddr);
  638. else
  639. ext2_set_bit(bit, kaddr);
  640. kunmap_atomic(kaddr, KM_USER0);
  641. PRINTK("set file bit %lu page %lu\n", bit, page->index);
  642. /* record page number so it gets flushed to disk when unplug occurs */
  643. set_page_attr(bitmap, page, BITMAP_PAGE_DIRTY);
  644. }
  645. /* this gets called when the md device is ready to unplug its underlying
  646. * (slave) device queues -- before we let any writes go down, we need to
  647. * sync the dirty pages of the bitmap file to disk */
  648. int bitmap_unplug(struct bitmap *bitmap)
  649. {
  650. unsigned long i, flags;
  651. int dirty, need_write;
  652. struct page *page;
  653. int wait = 0;
  654. int err;
  655. if (!bitmap)
  656. return 0;
  657. /* look at each page to see if there are any set bits that need to be
  658. * flushed out to disk */
  659. for (i = 0; i < bitmap->file_pages; i++) {
  660. spin_lock_irqsave(&bitmap->lock, flags);
  661. if (!bitmap->filemap) {
  662. spin_unlock_irqrestore(&bitmap->lock, flags);
  663. return 0;
  664. }
  665. page = bitmap->filemap[i];
  666. dirty = test_page_attr(bitmap, page, BITMAP_PAGE_DIRTY);
  667. need_write = test_page_attr(bitmap, page, BITMAP_PAGE_NEEDWRITE);
  668. clear_page_attr(bitmap, page, BITMAP_PAGE_DIRTY);
  669. clear_page_attr(bitmap, page, BITMAP_PAGE_NEEDWRITE);
  670. if (dirty)
  671. wait = 1;
  672. spin_unlock_irqrestore(&bitmap->lock, flags);
  673. if (dirty | need_write)
  674. err = write_page(bitmap, page, 0);
  675. }
  676. if (wait) { /* if any writes were performed, we need to wait on them */
  677. if (bitmap->file)
  678. wait_event(bitmap->write_wait,
  679. atomic_read(&bitmap->pending_writes)==0);
  680. else
  681. md_super_wait(bitmap->mddev);
  682. }
  683. if (bitmap->flags & BITMAP_WRITE_ERROR)
  684. bitmap_file_kick(bitmap);
  685. return 0;
  686. }
  687. static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);
  688. /* * bitmap_init_from_disk -- called at bitmap_create time to initialize
  689. * the in-memory bitmap from the on-disk bitmap -- also, sets up the
  690. * memory mapping of the bitmap file
  691. * Special cases:
  692. * if there's no bitmap file, or if the bitmap file had been
  693. * previously kicked from the array, we mark all the bits as
  694. * 1's in order to cause a full resync.
  695. *
  696. * We ignore all bits for sectors that end earlier than 'start'.
  697. * This is used when reading an out-of-date bitmap...
  698. */
  699. static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
  700. {
  701. unsigned long i, chunks, index, oldindex, bit;
  702. struct page *page = NULL, *oldpage = NULL;
  703. unsigned long num_pages, bit_cnt = 0;
  704. struct file *file;
  705. unsigned long bytes, offset;
  706. int outofdate;
  707. int ret = -ENOSPC;
  708. void *paddr;
  709. chunks = bitmap->chunks;
  710. file = bitmap->file;
  711. BUG_ON(!file && !bitmap->offset);
  712. #ifdef INJECT_FAULTS_3
  713. outofdate = 1;
  714. #else
  715. outofdate = bitmap->flags & BITMAP_STALE;
  716. #endif
  717. if (outofdate)
  718. printk(KERN_INFO "%s: bitmap file is out of date, doing full "
  719. "recovery\n", bmname(bitmap));
  720. bytes = (chunks + 7) / 8;
  721. num_pages = (bytes + sizeof(bitmap_super_t) + PAGE_SIZE - 1) / PAGE_SIZE;
  722. if (file && i_size_read(file->f_mapping->host) < bytes + sizeof(bitmap_super_t)) {
  723. printk(KERN_INFO "%s: bitmap file too short %lu < %lu\n",
  724. bmname(bitmap),
  725. (unsigned long) i_size_read(file->f_mapping->host),
  726. bytes + sizeof(bitmap_super_t));
  727. goto out;
  728. }
  729. ret = -ENOMEM;
  730. bitmap->filemap = kmalloc(sizeof(struct page *) * num_pages, GFP_KERNEL);
  731. if (!bitmap->filemap)
  732. goto out;
  733. /* We need 4 bits per page, rounded up to a multiple of sizeof(unsigned long) */
  734. bitmap->filemap_attr = kzalloc(
  735. (((num_pages*4/8)+sizeof(unsigned long)-1)
  736. /sizeof(unsigned long))
  737. *sizeof(unsigned long),
  738. GFP_KERNEL);
  739. if (!bitmap->filemap_attr)
  740. goto out;
  741. oldindex = ~0L;
  742. for (i = 0; i < chunks; i++) {
  743. int b;
  744. index = file_page_index(i);
  745. bit = file_page_offset(i);
  746. if (index != oldindex) { /* this is a new page, read it in */
  747. int count;
  748. /* unmap the old page, we're done with it */
  749. if (index == num_pages-1)
  750. count = bytes + sizeof(bitmap_super_t)
  751. - index * PAGE_SIZE;
  752. else
  753. count = PAGE_SIZE;
  754. if (index == 0) {
  755. /*
  756. * if we're here then the superblock page
  757. * contains some bits (PAGE_SIZE != sizeof sb)
  758. * we've already read it in, so just use it
  759. */
  760. page = bitmap->sb_page;
  761. offset = sizeof(bitmap_super_t);
  762. } else if (file) {
  763. page = read_page(file, index, bitmap, count);
  764. offset = 0;
  765. } else {
  766. page = read_sb_page(bitmap->mddev, bitmap->offset, index);
  767. offset = 0;
  768. }
  769. if (IS_ERR(page)) { /* read error */
  770. ret = PTR_ERR(page);
  771. goto out;
  772. }
  773. oldindex = index;
  774. oldpage = page;
  775. if (outofdate) {
  776. /*
  777. * if bitmap is out of date, dirty the
  778. * whole page and write it out
  779. */
  780. paddr = kmap_atomic(page, KM_USER0);
  781. memset(paddr + offset, 0xff,
  782. PAGE_SIZE - offset);
  783. kunmap_atomic(paddr, KM_USER0);
  784. ret = write_page(bitmap, page, 1);
  785. if (ret) {
  786. /* release, page not in filemap yet */
  787. put_page(page);
  788. goto out;
  789. }
  790. }
  791. bitmap->filemap[bitmap->file_pages++] = page;
  792. }
  793. paddr = kmap_atomic(page, KM_USER0);
  794. if (bitmap->flags & BITMAP_HOSTENDIAN)
  795. b = test_bit(bit, paddr);
  796. else
  797. b = ext2_test_bit(bit, paddr);
  798. kunmap_atomic(paddr, KM_USER0);
  799. if (b) {
  800. /* if the disk bit is set, set the memory bit */
  801. bitmap_set_memory_bits(bitmap, i << CHUNK_BLOCK_SHIFT(bitmap),
  802. ((i+1) << (CHUNK_BLOCK_SHIFT(bitmap)) >= start)
  803. );
  804. bit_cnt++;
  805. set_page_attr(bitmap, page, BITMAP_PAGE_CLEAN);
  806. }
  807. }
  808. /* everything went OK */
  809. ret = 0;
  810. bitmap_mask_state(bitmap, BITMAP_STALE, MASK_UNSET);
  811. if (bit_cnt) { /* Kick recovery if any bits were set */
  812. set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery);
  813. md_wakeup_thread(bitmap->mddev->thread);
  814. }
  815. out:
  816. printk(KERN_INFO "%s: bitmap initialized from disk: "
  817. "read %lu/%lu pages, set %lu bits, status: %d\n",
  818. bmname(bitmap), bitmap->file_pages, num_pages, bit_cnt, ret);
  819. return ret;
  820. }
  821. void bitmap_write_all(struct bitmap *bitmap)
  822. {
  823. /* We don't actually write all bitmap blocks here,
  824. * just flag them as needing to be written
  825. */
  826. int i;
  827. for (i=0; i < bitmap->file_pages; i++)
  828. set_page_attr(bitmap, bitmap->filemap[i],
  829. BITMAP_PAGE_NEEDWRITE);
  830. }
  831. static void bitmap_count_page(struct bitmap *bitmap, sector_t offset, int inc)
  832. {
  833. sector_t chunk = offset >> CHUNK_BLOCK_SHIFT(bitmap);
  834. unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
  835. bitmap->bp[page].count += inc;
  836. /*
  837. if (page == 0) printk("count page 0, offset %llu: %d gives %d\n",
  838. (unsigned long long)offset, inc, bitmap->bp[page].count);
  839. */
  840. bitmap_checkfree(bitmap, page);
  841. }
  842. static bitmap_counter_t *bitmap_get_counter(struct bitmap *bitmap,
  843. sector_t offset, int *blocks,
  844. int create);
  845. /*
  846. * bitmap daemon -- periodically wakes up to clean bits and flush pages
  847. * out to disk
  848. */
  849. int bitmap_daemon_work(struct bitmap *bitmap)
  850. {
  851. unsigned long j;
  852. unsigned long flags;
  853. struct page *page = NULL, *lastpage = NULL;
  854. int err = 0;
  855. int blocks;
  856. void *paddr;
  857. if (bitmap == NULL)
  858. return 0;
  859. if (time_before(jiffies, bitmap->daemon_lastrun + bitmap->daemon_sleep*HZ))
  860. return 0;
  861. bitmap->daemon_lastrun = jiffies;
  862. for (j = 0; j < bitmap->chunks; j++) {
  863. bitmap_counter_t *bmc;
  864. spin_lock_irqsave(&bitmap->lock, flags);
  865. if (!bitmap->filemap) {
  866. /* error or shutdown */
  867. spin_unlock_irqrestore(&bitmap->lock, flags);
  868. break;
  869. }
  870. page = filemap_get_page(bitmap, j);
  871. if (page != lastpage) {
  872. /* skip this page unless it's marked as needing cleaning */
  873. if (!test_page_attr(bitmap, page, BITMAP_PAGE_CLEAN)) {
  874. int need_write = test_page_attr(bitmap, page,
  875. BITMAP_PAGE_NEEDWRITE);
  876. if (need_write)
  877. clear_page_attr(bitmap, page, BITMAP_PAGE_NEEDWRITE);
  878. spin_unlock_irqrestore(&bitmap->lock, flags);
  879. if (need_write) {
  880. switch (write_page(bitmap, page, 0)) {
  881. case 0:
  882. break;
  883. default:
  884. bitmap_file_kick(bitmap);
  885. }
  886. }
  887. continue;
  888. }
  889. /* grab the new page, sync and release the old */
  890. if (lastpage != NULL) {
  891. if (test_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE)) {
  892. clear_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE);
  893. spin_unlock_irqrestore(&bitmap->lock, flags);
  894. err = write_page(bitmap, lastpage, 0);
  895. } else {
  896. set_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE);
  897. spin_unlock_irqrestore(&bitmap->lock, flags);
  898. }
  899. if (err)
  900. bitmap_file_kick(bitmap);
  901. } else
  902. spin_unlock_irqrestore(&bitmap->lock, flags);
  903. lastpage = page;
  904. /*
  905. printk("bitmap clean at page %lu\n", j);
  906. */
  907. spin_lock_irqsave(&bitmap->lock, flags);
  908. clear_page_attr(bitmap, page, BITMAP_PAGE_CLEAN);
  909. }
  910. bmc = bitmap_get_counter(bitmap, j << CHUNK_BLOCK_SHIFT(bitmap),
  911. &blocks, 0);
  912. if (bmc) {
  913. /*
  914. if (j < 100) printk("bitmap: j=%lu, *bmc = 0x%x\n", j, *bmc);
  915. */
  916. if (*bmc == 2) {
  917. *bmc=1; /* maybe clear the bit next time */
  918. set_page_attr(bitmap, page, BITMAP_PAGE_CLEAN);
  919. } else if (*bmc == 1) {
  920. /* we can clear the bit */
  921. *bmc = 0;
  922. bitmap_count_page(bitmap, j << CHUNK_BLOCK_SHIFT(bitmap),
  923. -1);
  924. /* clear the bit */
  925. paddr = kmap_atomic(page, KM_USER0);
  926. if (bitmap->flags & BITMAP_HOSTENDIAN)
  927. clear_bit(file_page_offset(j), paddr);
  928. else
  929. ext2_clear_bit(file_page_offset(j), paddr);
  930. kunmap_atomic(paddr, KM_USER0);
  931. }
  932. }
  933. spin_unlock_irqrestore(&bitmap->lock, flags);
  934. }
  935. /* now sync the final page */
  936. if (lastpage != NULL) {
  937. spin_lock_irqsave(&bitmap->lock, flags);
  938. if (test_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE)) {
  939. clear_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE);
  940. spin_unlock_irqrestore(&bitmap->lock, flags);
  941. err = write_page(bitmap, lastpage, 0);
  942. } else {
  943. set_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE);
  944. spin_unlock_irqrestore(&bitmap->lock, flags);
  945. }
  946. }
  947. return err;
  948. }
  949. static bitmap_counter_t *bitmap_get_counter(struct bitmap *bitmap,
  950. sector_t offset, int *blocks,
  951. int create)
  952. {
  953. /* If 'create', we might release the lock and reclaim it.
  954. * The lock must have been taken with interrupts enabled.
  955. * If !create, we don't release the lock.
  956. */
  957. sector_t chunk = offset >> CHUNK_BLOCK_SHIFT(bitmap);
  958. unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
  959. unsigned long pageoff = (chunk & PAGE_COUNTER_MASK) << COUNTER_BYTE_SHIFT;
  960. sector_t csize;
  961. if (bitmap_checkpage(bitmap, page, create) < 0) {
  962. csize = ((sector_t)1) << (CHUNK_BLOCK_SHIFT(bitmap));
  963. *blocks = csize - (offset & (csize- 1));
  964. return NULL;
  965. }
  966. /* now locked ... */
  967. if (bitmap->bp[page].hijacked) { /* hijacked pointer */
  968. /* should we use the first or second counter field
  969. * of the hijacked pointer? */
  970. int hi = (pageoff > PAGE_COUNTER_MASK);
  971. csize = ((sector_t)1) << (CHUNK_BLOCK_SHIFT(bitmap) +
  972. PAGE_COUNTER_SHIFT - 1);
  973. *blocks = csize - (offset & (csize- 1));
  974. return &((bitmap_counter_t *)
  975. &bitmap->bp[page].map)[hi];
  976. } else { /* page is allocated */
  977. csize = ((sector_t)1) << (CHUNK_BLOCK_SHIFT(bitmap));
  978. *blocks = csize - (offset & (csize- 1));
  979. return (bitmap_counter_t *)
  980. &(bitmap->bp[page].map[pageoff]);
  981. }
  982. }
  983. int bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors, int behind)
  984. {
  985. if (!bitmap) return 0;
  986. if (behind) {
  987. atomic_inc(&bitmap->behind_writes);
  988. PRINTK(KERN_DEBUG "inc write-behind count %d/%d\n",
  989. atomic_read(&bitmap->behind_writes), bitmap->max_write_behind);
  990. }
  991. while (sectors) {
  992. int blocks;
  993. bitmap_counter_t *bmc;
  994. spin_lock_irq(&bitmap->lock);
  995. bmc = bitmap_get_counter(bitmap, offset, &blocks, 1);
  996. if (!bmc) {
  997. spin_unlock_irq(&bitmap->lock);
  998. return 0;
  999. }
  1000. switch(*bmc) {
  1001. case 0:
  1002. bitmap_file_set_bit(bitmap, offset);
  1003. bitmap_count_page(bitmap,offset, 1);
  1004. blk_plug_device(bitmap->mddev->queue);
  1005. /* fall through */
  1006. case 1:
  1007. *bmc = 2;
  1008. }
  1009. BUG_ON((*bmc & COUNTER_MAX) == COUNTER_MAX);
  1010. (*bmc)++;
  1011. spin_unlock_irq(&bitmap->lock);
  1012. offset += blocks;
  1013. if (sectors > blocks)
  1014. sectors -= blocks;
  1015. else sectors = 0;
  1016. }
  1017. return 0;
  1018. }
  1019. void bitmap_endwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors,
  1020. int success, int behind)
  1021. {
  1022. if (!bitmap) return;
  1023. if (behind) {
  1024. atomic_dec(&bitmap->behind_writes);
  1025. PRINTK(KERN_DEBUG "dec write-behind count %d/%d\n",
  1026. atomic_read(&bitmap->behind_writes), bitmap->max_write_behind);
  1027. }
  1028. while (sectors) {
  1029. int blocks;
  1030. unsigned long flags;
  1031. bitmap_counter_t *bmc;
  1032. spin_lock_irqsave(&bitmap->lock, flags);
  1033. bmc = bitmap_get_counter(bitmap, offset, &blocks, 0);
  1034. if (!bmc) {
  1035. spin_unlock_irqrestore(&bitmap->lock, flags);
  1036. return;
  1037. }
  1038. if (!success && ! (*bmc & NEEDED_MASK))
  1039. *bmc |= NEEDED_MASK;
  1040. (*bmc)--;
  1041. if (*bmc <= 2) {
  1042. set_page_attr(bitmap,
  1043. filemap_get_page(bitmap, offset >> CHUNK_BLOCK_SHIFT(bitmap)),
  1044. BITMAP_PAGE_CLEAN);
  1045. }
  1046. spin_unlock_irqrestore(&bitmap->lock, flags);
  1047. offset += blocks;
  1048. if (sectors > blocks)
  1049. sectors -= blocks;
  1050. else sectors = 0;
  1051. }
  1052. }
  1053. int bitmap_start_sync(struct bitmap *bitmap, sector_t offset, int *blocks,
  1054. int degraded)
  1055. {
  1056. bitmap_counter_t *bmc;
  1057. int rv;
  1058. if (bitmap == NULL) {/* FIXME or bitmap set as 'failed' */
  1059. *blocks = 1024;
  1060. return 1; /* always resync if no bitmap */
  1061. }
  1062. spin_lock_irq(&bitmap->lock);
  1063. bmc = bitmap_get_counter(bitmap, offset, blocks, 0);
  1064. rv = 0;
  1065. if (bmc) {
  1066. /* locked */
  1067. if (RESYNC(*bmc))
  1068. rv = 1;
  1069. else if (NEEDED(*bmc)) {
  1070. rv = 1;
  1071. if (!degraded) { /* don't set/clear bits if degraded */
  1072. *bmc |= RESYNC_MASK;
  1073. *bmc &= ~NEEDED_MASK;
  1074. }
  1075. }
  1076. }
  1077. spin_unlock_irq(&bitmap->lock);
  1078. return rv;
  1079. }
  1080. void bitmap_end_sync(struct bitmap *bitmap, sector_t offset, int *blocks, int aborted)
  1081. {
  1082. bitmap_counter_t *bmc;
  1083. unsigned long flags;
  1084. /*
  1085. if (offset == 0) printk("bitmap_end_sync 0 (%d)\n", aborted);
  1086. */ if (bitmap == NULL) {
  1087. *blocks = 1024;
  1088. return;
  1089. }
  1090. spin_lock_irqsave(&bitmap->lock, flags);
  1091. bmc = bitmap_get_counter(bitmap, offset, blocks, 0);
  1092. if (bmc == NULL)
  1093. goto unlock;
  1094. /* locked */
  1095. /*
  1096. if (offset == 0) printk("bitmap_end sync found 0x%x, blocks %d\n", *bmc, *blocks);
  1097. */
  1098. if (RESYNC(*bmc)) {
  1099. *bmc &= ~RESYNC_MASK;
  1100. if (!NEEDED(*bmc) && aborted)
  1101. *bmc |= NEEDED_MASK;
  1102. else {
  1103. if (*bmc <= 2) {
  1104. set_page_attr(bitmap,
  1105. filemap_get_page(bitmap, offset >> CHUNK_BLOCK_SHIFT(bitmap)),
  1106. BITMAP_PAGE_CLEAN);
  1107. }
  1108. }
  1109. }
  1110. unlock:
  1111. spin_unlock_irqrestore(&bitmap->lock, flags);
  1112. }
  1113. void bitmap_close_sync(struct bitmap *bitmap)
  1114. {
  1115. /* Sync has finished, and any bitmap chunks that weren't synced
  1116. * properly have been aborted. It remains to us to clear the
  1117. * RESYNC bit wherever it is still on
  1118. */
  1119. sector_t sector = 0;
  1120. int blocks;
  1121. if (!bitmap) return;
  1122. while (sector < bitmap->mddev->resync_max_sectors) {
  1123. bitmap_end_sync(bitmap, sector, &blocks, 0);
  1124. /*
  1125. if (sector < 500) printk("bitmap_close_sync: sec %llu blks %d\n",
  1126. (unsigned long long)sector, blocks);
  1127. */ sector += blocks;
  1128. }
  1129. }
  1130. static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed)
  1131. {
  1132. /* For each chunk covered by any of these sectors, set the
  1133. * counter to 1 and set resync_needed. They should all
  1134. * be 0 at this point
  1135. */
  1136. int secs;
  1137. bitmap_counter_t *bmc;
  1138. spin_lock_irq(&bitmap->lock);
  1139. bmc = bitmap_get_counter(bitmap, offset, &secs, 1);
  1140. if (!bmc) {
  1141. spin_unlock_irq(&bitmap->lock);
  1142. return;
  1143. }
  1144. if (! *bmc) {
  1145. struct page *page;
  1146. *bmc = 1 | (needed?NEEDED_MASK:0);
  1147. bitmap_count_page(bitmap, offset, 1);
  1148. page = filemap_get_page(bitmap, offset >> CHUNK_BLOCK_SHIFT(bitmap));
  1149. set_page_attr(bitmap, page, BITMAP_PAGE_CLEAN);
  1150. }
  1151. spin_unlock_irq(&bitmap->lock);
  1152. }
  1153. /* dirty the memory and file bits for bitmap chunks "s" to "e" */
  1154. void bitmap_dirty_bits(struct bitmap *bitmap, unsigned long s, unsigned long e)
  1155. {
  1156. unsigned long chunk;
  1157. for (chunk = s; chunk <= e; chunk++) {
  1158. sector_t sec = chunk << CHUNK_BLOCK_SHIFT(bitmap);
  1159. bitmap_set_memory_bits(bitmap, sec, 1);
  1160. bitmap_file_set_bit(bitmap, sec);
  1161. }
  1162. }
  1163. /*
  1164. * flush out any pending updates
  1165. */
  1166. void bitmap_flush(mddev_t *mddev)
  1167. {
  1168. struct bitmap *bitmap = mddev->bitmap;
  1169. int sleep;
  1170. if (!bitmap) /* there was no bitmap */
  1171. return;
  1172. /* run the daemon_work three time to ensure everything is flushed
  1173. * that can be
  1174. */
  1175. sleep = bitmap->daemon_sleep;
  1176. bitmap->daemon_sleep = 0;
  1177. bitmap_daemon_work(bitmap);
  1178. bitmap_daemon_work(bitmap);
  1179. bitmap_daemon_work(bitmap);
  1180. bitmap->daemon_sleep = sleep;
  1181. bitmap_update_sb(bitmap);
  1182. }
  1183. /*
  1184. * free memory that was allocated
  1185. */
  1186. static void bitmap_free(struct bitmap *bitmap)
  1187. {
  1188. unsigned long k, pages;
  1189. struct bitmap_page *bp;
  1190. if (!bitmap) /* there was no bitmap */
  1191. return;
  1192. /* release the bitmap file and kill the daemon */
  1193. bitmap_file_put(bitmap);
  1194. bp = bitmap->bp;
  1195. pages = bitmap->pages;
  1196. /* free all allocated memory */
  1197. if (bp) /* deallocate the page memory */
  1198. for (k = 0; k < pages; k++)
  1199. if (bp[k].map && !bp[k].hijacked)
  1200. kfree(bp[k].map);
  1201. kfree(bp);
  1202. kfree(bitmap);
  1203. }
  1204. void bitmap_destroy(mddev_t *mddev)
  1205. {
  1206. struct bitmap *bitmap = mddev->bitmap;
  1207. if (!bitmap) /* there was no bitmap */
  1208. return;
  1209. mddev->bitmap = NULL; /* disconnect from the md device */
  1210. if (mddev->thread)
  1211. mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
  1212. bitmap_free(bitmap);
  1213. }
  1214. /*
  1215. * initialize the bitmap structure
  1216. * if this returns an error, bitmap_destroy must be called to do clean up
  1217. */
  1218. int bitmap_create(mddev_t *mddev)
  1219. {
  1220. struct bitmap *bitmap;
  1221. unsigned long blocks = mddev->resync_max_sectors;
  1222. unsigned long chunks;
  1223. unsigned long pages;
  1224. struct file *file = mddev->bitmap_file;
  1225. int err;
  1226. sector_t start;
  1227. BUILD_BUG_ON(sizeof(bitmap_super_t) != 256);
  1228. if (!file && !mddev->bitmap_offset) /* bitmap disabled, nothing to do */
  1229. return 0;
  1230. BUG_ON(file && mddev->bitmap_offset);
  1231. bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL);
  1232. if (!bitmap)
  1233. return -ENOMEM;
  1234. spin_lock_init(&bitmap->lock);
  1235. atomic_set(&bitmap->pending_writes, 0);
  1236. init_waitqueue_head(&bitmap->write_wait);
  1237. bitmap->mddev = mddev;
  1238. bitmap->file = file;
  1239. bitmap->offset = mddev->bitmap_offset;
  1240. if (file) {
  1241. get_file(file);
  1242. do_sync_file_range(file, 0, LLONG_MAX,
  1243. SYNC_FILE_RANGE_WAIT_BEFORE |
  1244. SYNC_FILE_RANGE_WRITE |
  1245. SYNC_FILE_RANGE_WAIT_AFTER);
  1246. }
  1247. /* read superblock from bitmap file (this sets bitmap->chunksize) */
  1248. err = bitmap_read_sb(bitmap);
  1249. if (err)
  1250. goto error;
  1251. bitmap->chunkshift = ffz(~bitmap->chunksize);
  1252. /* now that chunksize and chunkshift are set, we can use these macros */
  1253. chunks = (blocks + CHUNK_BLOCK_RATIO(bitmap) - 1) /
  1254. CHUNK_BLOCK_RATIO(bitmap);
  1255. pages = (chunks + PAGE_COUNTER_RATIO - 1) / PAGE_COUNTER_RATIO;
  1256. BUG_ON(!pages);
  1257. bitmap->chunks = chunks;
  1258. bitmap->pages = pages;
  1259. bitmap->missing_pages = pages;
  1260. bitmap->counter_bits = COUNTER_BITS;
  1261. bitmap->syncchunk = ~0UL;
  1262. #ifdef INJECT_FATAL_FAULT_1
  1263. bitmap->bp = NULL;
  1264. #else
  1265. bitmap->bp = kzalloc(pages * sizeof(*bitmap->bp), GFP_KERNEL);
  1266. #endif
  1267. err = -ENOMEM;
  1268. if (!bitmap->bp)
  1269. goto error;
  1270. /* now that we have some pages available, initialize the in-memory
  1271. * bitmap from the on-disk bitmap */
  1272. start = 0;
  1273. if (mddev->degraded == 0
  1274. || bitmap->events_cleared == mddev->events)
  1275. /* no need to keep dirty bits to optimise a re-add of a missing device */
  1276. start = mddev->recovery_cp;
  1277. err = bitmap_init_from_disk(bitmap, start);
  1278. if (err)
  1279. goto error;
  1280. printk(KERN_INFO "created bitmap (%lu pages) for device %s\n",
  1281. pages, bmname(bitmap));
  1282. mddev->bitmap = bitmap;
  1283. mddev->thread->timeout = bitmap->daemon_sleep * HZ;
  1284. return bitmap_update_sb(bitmap);
  1285. error:
  1286. bitmap_free(bitmap);
  1287. return err;
  1288. }
  1289. /* the bitmap API -- for raid personalities */
  1290. EXPORT_SYMBOL(bitmap_startwrite);
  1291. EXPORT_SYMBOL(bitmap_endwrite);
  1292. EXPORT_SYMBOL(bitmap_start_sync);
  1293. EXPORT_SYMBOL(bitmap_end_sync);
  1294. EXPORT_SYMBOL(bitmap_unplug);
  1295. EXPORT_SYMBOL(bitmap_close_sync);