bitmap.c 39 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498
  1. /*
  2. * bitmap.c two-level bitmap (C) Peter T. Breuer (ptb@ot.uc3m.es) 2003
  3. *
  4. * bitmap_create - sets up the bitmap structure
  5. * bitmap_destroy - destroys the bitmap structure
  6. *
  7. * additions, Copyright (C) 2003-2004, Paul Clements, SteelEye Technology, Inc.:
  8. * - added disk storage for bitmap
  9. * - changes to allow various bitmap chunk sizes
  10. * - added bitmap daemon (to asynchronously clear bitmap bits from disk)
  11. */
  12. /*
  13. * Still to do:
  14. *
  15. * flush after percent set rather than just time based. (maybe both).
  16. * wait if count gets too high, wake when it drops to half.
  17. * allow bitmap to be mirrored with superblock (before or after...)
  18. * allow hot-add to re-instate a current device.
  19. * allow hot-add of bitmap after quiessing device
  20. */
  21. #include <linux/module.h>
  22. #include <linux/version.h>
  23. #include <linux/errno.h>
  24. #include <linux/slab.h>
  25. #include <linux/init.h>
  26. #include <linux/config.h>
  27. #include <linux/timer.h>
  28. #include <linux/sched.h>
  29. #include <linux/list.h>
  30. #include <linux/file.h>
  31. #include <linux/mount.h>
  32. #include <linux/buffer_head.h>
  33. #include <linux/raid/md.h>
  34. #include <linux/raid/bitmap.h>
  35. /* debug macros */
  36. #define DEBUG 0
  37. #if DEBUG
  38. /* these are for debugging purposes only! */
  39. /* define one and only one of these */
  40. #define INJECT_FAULTS_1 0 /* cause bitmap_alloc_page to fail always */
  41. #define INJECT_FAULTS_2 0 /* cause bitmap file to be kicked when first bit set*/
  42. #define INJECT_FAULTS_3 0 /* treat bitmap file as kicked at init time */
  43. #define INJECT_FAULTS_4 0 /* undef */
  44. #define INJECT_FAULTS_5 0 /* undef */
  45. #define INJECT_FAULTS_6 0
  46. /* if these are defined, the driver will fail! debug only */
  47. #define INJECT_FATAL_FAULT_1 0 /* fail kmalloc, causing bitmap_create to fail */
  48. #define INJECT_FATAL_FAULT_2 0 /* undef */
  49. #define INJECT_FATAL_FAULT_3 0 /* undef */
  50. #endif
  51. //#define DPRINTK PRINTK /* set this NULL to avoid verbose debug output */
  52. #define DPRINTK(x...) do { } while(0)
  53. #ifndef PRINTK
  54. # if DEBUG > 0
  55. # define PRINTK(x...) printk(KERN_DEBUG x)
  56. # else
  57. # define PRINTK(x...)
  58. # endif
  59. #endif
  60. static inline char * bmname(struct bitmap *bitmap)
  61. {
  62. return bitmap->mddev ? mdname(bitmap->mddev) : "mdX";
  63. }
  64. /*
  65. * test if the bitmap is active
  66. */
  67. int bitmap_active(struct bitmap *bitmap)
  68. {
  69. unsigned long flags;
  70. int res = 0;
  71. if (!bitmap)
  72. return res;
  73. spin_lock_irqsave(&bitmap->lock, flags);
  74. res = bitmap->flags & BITMAP_ACTIVE;
  75. spin_unlock_irqrestore(&bitmap->lock, flags);
  76. return res;
  77. }
  78. #define WRITE_POOL_SIZE 256
  79. /* mempool for queueing pending writes on the bitmap file */
  80. static void *write_pool_alloc(unsigned int gfp_flags, void *data)
  81. {
  82. return kmalloc(sizeof(struct page_list), gfp_flags);
  83. }
  84. static void write_pool_free(void *ptr, void *data)
  85. {
  86. kfree(ptr);
  87. }
  88. /*
  89. * just a placeholder - calls kmalloc for bitmap pages
  90. */
  91. static unsigned char *bitmap_alloc_page(struct bitmap *bitmap)
  92. {
  93. unsigned char *page;
  94. #if INJECT_FAULTS_1
  95. page = NULL;
  96. #else
  97. page = kmalloc(PAGE_SIZE, GFP_NOIO);
  98. #endif
  99. if (!page)
  100. printk("%s: bitmap_alloc_page FAILED\n", bmname(bitmap));
  101. else
  102. printk("%s: bitmap_alloc_page: allocated page at %p\n",
  103. bmname(bitmap), page);
  104. return page;
  105. }
  106. /*
  107. * for now just a placeholder -- just calls kfree for bitmap pages
  108. */
  109. static void bitmap_free_page(struct bitmap *bitmap, unsigned char *page)
  110. {
  111. PRINTK("%s: bitmap_free_page: free page %p\n", bmname(bitmap), page);
  112. kfree(page);
  113. }
  114. /*
  115. * check a page and, if necessary, allocate it (or hijack it if the alloc fails)
  116. *
  117. * 1) check to see if this page is allocated, if it's not then try to alloc
  118. * 2) if the alloc fails, set the page's hijacked flag so we'll use the
  119. * page pointer directly as a counter
  120. *
  121. * if we find our page, we increment the page's refcount so that it stays
  122. * allocated while we're using it
  123. */
  124. static int bitmap_checkpage(struct bitmap *bitmap, unsigned long page, int create)
  125. {
  126. unsigned char *mappage;
  127. if (page >= bitmap->pages) {
  128. printk(KERN_ALERT
  129. "%s: invalid bitmap page request: %lu (> %lu)\n",
  130. bmname(bitmap), page, bitmap->pages-1);
  131. return -EINVAL;
  132. }
  133. if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */
  134. return 0;
  135. if (bitmap->bp[page].map) /* page is already allocated, just return */
  136. return 0;
  137. if (!create)
  138. return -ENOENT;
  139. spin_unlock_irq(&bitmap->lock);
  140. /* this page has not been allocated yet */
  141. if ((mappage = bitmap_alloc_page(bitmap)) == NULL) {
  142. PRINTK("%s: bitmap map page allocation failed, hijacking\n",
  143. bmname(bitmap));
  144. /* failed - set the hijacked flag so that we can use the
  145. * pointer as a counter */
  146. spin_lock_irq(&bitmap->lock);
  147. if (!bitmap->bp[page].map)
  148. bitmap->bp[page].hijacked = 1;
  149. goto out;
  150. }
  151. /* got a page */
  152. spin_lock_irq(&bitmap->lock);
  153. /* recheck the page */
  154. if (bitmap->bp[page].map || bitmap->bp[page].hijacked) {
  155. /* somebody beat us to getting the page */
  156. bitmap_free_page(bitmap, mappage);
  157. return 0;
  158. }
  159. /* no page was in place and we have one, so install it */
  160. memset(mappage, 0, PAGE_SIZE);
  161. bitmap->bp[page].map = mappage;
  162. bitmap->missing_pages--;
  163. out:
  164. return 0;
  165. }
  166. /* if page is completely empty, put it back on the free list, or dealloc it */
  167. /* if page was hijacked, unmark the flag so it might get alloced next time */
  168. /* Note: lock should be held when calling this */
  169. static inline void bitmap_checkfree(struct bitmap *bitmap, unsigned long page)
  170. {
  171. char *ptr;
  172. if (bitmap->bp[page].count) /* page is still busy */
  173. return;
  174. /* page is no longer in use, it can be released */
  175. if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */
  176. bitmap->bp[page].hijacked = 0;
  177. bitmap->bp[page].map = NULL;
  178. return;
  179. }
  180. /* normal case, free the page */
  181. #if 0
  182. /* actually ... let's not. We will probably need the page again exactly when
  183. * memory is tight and we are flusing to disk
  184. */
  185. return;
  186. #else
  187. ptr = bitmap->bp[page].map;
  188. bitmap->bp[page].map = NULL;
  189. bitmap->missing_pages++;
  190. bitmap_free_page(bitmap, ptr);
  191. return;
  192. #endif
  193. }
  194. /*
  195. * bitmap file handling - read and write the bitmap file and its superblock
  196. */
  197. /* copy the pathname of a file to a buffer */
  198. char *file_path(struct file *file, char *buf, int count)
  199. {
  200. struct dentry *d;
  201. struct vfsmount *v;
  202. if (!buf)
  203. return NULL;
  204. d = file->f_dentry;
  205. v = file->f_vfsmnt;
  206. buf = d_path(d, v, buf, count);
  207. return IS_ERR(buf) ? NULL : buf;
  208. }
  209. /*
  210. * basic page I/O operations
  211. */
  212. /*
  213. * write out a page
  214. */
  215. static int write_page(struct bitmap *bitmap, struct page *page, int wait)
  216. {
  217. int ret = -ENOMEM;
  218. lock_page(page);
  219. ret = page->mapping->a_ops->prepare_write(NULL, page, 0, PAGE_SIZE);
  220. if (!ret)
  221. ret = page->mapping->a_ops->commit_write(NULL, page, 0,
  222. PAGE_SIZE);
  223. if (ret) {
  224. unlock_page(page);
  225. return ret;
  226. }
  227. set_page_dirty(page); /* force it to be written out */
  228. if (!wait) {
  229. /* add to list to be waited for by daemon */
  230. struct page_list *item = mempool_alloc(bitmap->write_pool, GFP_NOIO);
  231. item->page = page;
  232. page_cache_get(page);
  233. spin_lock(&bitmap->write_lock);
  234. list_add(&item->list, &bitmap->complete_pages);
  235. spin_unlock(&bitmap->write_lock);
  236. md_wakeup_thread(bitmap->writeback_daemon);
  237. }
  238. return write_one_page(page, wait);
  239. }
  240. /* read a page from a file, pinning it into cache, and return bytes_read */
  241. static struct page *read_page(struct file *file, unsigned long index,
  242. unsigned long *bytes_read)
  243. {
  244. struct inode *inode = file->f_mapping->host;
  245. struct page *page = NULL;
  246. loff_t isize = i_size_read(inode);
  247. unsigned long end_index = isize >> PAGE_CACHE_SHIFT;
  248. PRINTK("read bitmap file (%dB @ %Lu)\n", (int)PAGE_CACHE_SIZE,
  249. (unsigned long long)index << PAGE_CACHE_SHIFT);
  250. page = read_cache_page(inode->i_mapping, index,
  251. (filler_t *)inode->i_mapping->a_ops->readpage, file);
  252. if (IS_ERR(page))
  253. goto out;
  254. wait_on_page_locked(page);
  255. if (!PageUptodate(page) || PageError(page)) {
  256. page_cache_release(page);
  257. page = ERR_PTR(-EIO);
  258. goto out;
  259. }
  260. if (index > end_index) /* we have read beyond EOF */
  261. *bytes_read = 0;
  262. else if (index == end_index) /* possible short read */
  263. *bytes_read = isize & ~PAGE_CACHE_MASK;
  264. else
  265. *bytes_read = PAGE_CACHE_SIZE; /* got a full page */
  266. out:
  267. if (IS_ERR(page))
  268. printk(KERN_ALERT "md: bitmap read error: (%dB @ %Lu): %ld\n",
  269. (int)PAGE_CACHE_SIZE,
  270. (unsigned long long)index << PAGE_CACHE_SHIFT,
  271. PTR_ERR(page));
  272. return page;
  273. }
  274. /*
  275. * bitmap file superblock operations
  276. */
  277. /* update the event counter and sync the superblock to disk */
  278. int bitmap_update_sb(struct bitmap *bitmap)
  279. {
  280. bitmap_super_t *sb;
  281. unsigned long flags;
  282. if (!bitmap || !bitmap->mddev) /* no bitmap for this array */
  283. return 0;
  284. spin_lock_irqsave(&bitmap->lock, flags);
  285. if (!bitmap->sb_page) { /* no superblock */
  286. spin_unlock_irqrestore(&bitmap->lock, flags);
  287. return 0;
  288. }
  289. spin_unlock_irqrestore(&bitmap->lock, flags);
  290. sb = (bitmap_super_t *)kmap(bitmap->sb_page);
  291. sb->events = cpu_to_le64(bitmap->mddev->events);
  292. if (!bitmap->mddev->degraded)
  293. sb->events_cleared = cpu_to_le64(bitmap->mddev->events);
  294. kunmap(bitmap->sb_page);
  295. return write_page(bitmap, bitmap->sb_page, 0);
  296. }
  297. /* print out the bitmap file superblock */
  298. void bitmap_print_sb(struct bitmap *bitmap)
  299. {
  300. bitmap_super_t *sb;
  301. if (!bitmap || !bitmap->sb_page)
  302. return;
  303. sb = (bitmap_super_t *)kmap(bitmap->sb_page);
  304. printk(KERN_DEBUG "%s: bitmap file superblock:\n", bmname(bitmap));
  305. printk(KERN_DEBUG " magic: %08x\n", le32_to_cpu(sb->magic));
  306. printk(KERN_DEBUG " version: %d\n", le32_to_cpu(sb->version));
  307. printk(KERN_DEBUG " uuid: %08x.%08x.%08x.%08x\n",
  308. *(__u32 *)(sb->uuid+0),
  309. *(__u32 *)(sb->uuid+4),
  310. *(__u32 *)(sb->uuid+8),
  311. *(__u32 *)(sb->uuid+12));
  312. printk(KERN_DEBUG " events: %llu\n",
  313. (unsigned long long) le64_to_cpu(sb->events));
  314. printk(KERN_DEBUG "events cleared: %llu\n",
  315. (unsigned long long) le64_to_cpu(sb->events_cleared));
  316. printk(KERN_DEBUG " state: %08x\n", le32_to_cpu(sb->state));
  317. printk(KERN_DEBUG " chunksize: %d B\n", le32_to_cpu(sb->chunksize));
  318. printk(KERN_DEBUG " daemon sleep: %ds\n", le32_to_cpu(sb->daemon_sleep));
  319. printk(KERN_DEBUG " sync size: %llu KB\n",
  320. (unsigned long long)le64_to_cpu(sb->sync_size)/2);
  321. kunmap(bitmap->sb_page);
  322. }
  323. /* read the superblock from the bitmap file and initialize some bitmap fields */
  324. static int bitmap_read_sb(struct bitmap *bitmap)
  325. {
  326. char *reason = NULL;
  327. bitmap_super_t *sb;
  328. unsigned long chunksize, daemon_sleep;
  329. unsigned long bytes_read;
  330. unsigned long long events;
  331. int err = -EINVAL;
  332. /* page 0 is the superblock, read it... */
  333. bitmap->sb_page = read_page(bitmap->file, 0, &bytes_read);
  334. if (IS_ERR(bitmap->sb_page)) {
  335. err = PTR_ERR(bitmap->sb_page);
  336. bitmap->sb_page = NULL;
  337. return err;
  338. }
  339. sb = (bitmap_super_t *)kmap(bitmap->sb_page);
  340. if (bytes_read < sizeof(*sb)) { /* short read */
  341. printk(KERN_INFO "%s: bitmap file superblock truncated\n",
  342. bmname(bitmap));
  343. err = -ENOSPC;
  344. goto out;
  345. }
  346. chunksize = le32_to_cpu(sb->chunksize);
  347. daemon_sleep = le32_to_cpu(sb->daemon_sleep);
  348. /* verify that the bitmap-specific fields are valid */
  349. if (sb->magic != cpu_to_le32(BITMAP_MAGIC))
  350. reason = "bad magic";
  351. else if (sb->version != cpu_to_le32(BITMAP_MAJOR))
  352. reason = "unrecognized superblock version";
  353. else if (chunksize < 512 || chunksize > (1024 * 1024 * 4))
  354. reason = "bitmap chunksize out of range (512B - 4MB)";
  355. else if ((1 << ffz(~chunksize)) != chunksize)
  356. reason = "bitmap chunksize not a power of 2";
  357. else if (daemon_sleep < 1 || daemon_sleep > 15)
  358. reason = "daemon sleep period out of range";
  359. if (reason) {
  360. printk(KERN_INFO "%s: invalid bitmap file superblock: %s\n",
  361. bmname(bitmap), reason);
  362. goto out;
  363. }
  364. /* keep the array size field of the bitmap superblock up to date */
  365. sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
  366. if (!bitmap->mddev->persistent)
  367. goto success;
  368. /*
  369. * if we have a persistent array superblock, compare the
  370. * bitmap's UUID and event counter to the mddev's
  371. */
  372. if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) {
  373. printk(KERN_INFO "%s: bitmap superblock UUID mismatch\n",
  374. bmname(bitmap));
  375. goto out;
  376. }
  377. events = le64_to_cpu(sb->events);
  378. if (events < bitmap->mddev->events) {
  379. printk(KERN_INFO "%s: bitmap file is out of date (%llu < %llu) "
  380. "-- forcing full recovery\n", bmname(bitmap), events,
  381. (unsigned long long) bitmap->mddev->events);
  382. sb->state |= BITMAP_STALE;
  383. }
  384. success:
  385. /* assign fields using values from superblock */
  386. bitmap->chunksize = chunksize;
  387. bitmap->daemon_sleep = daemon_sleep;
  388. bitmap->flags |= sb->state;
  389. bitmap->events_cleared = le64_to_cpu(sb->events_cleared);
  390. err = 0;
  391. out:
  392. kunmap(bitmap->sb_page);
  393. if (err)
  394. bitmap_print_sb(bitmap);
  395. return err;
  396. }
  397. enum bitmap_mask_op {
  398. MASK_SET,
  399. MASK_UNSET
  400. };
  401. /* record the state of the bitmap in the superblock */
  402. static void bitmap_mask_state(struct bitmap *bitmap, enum bitmap_state bits,
  403. enum bitmap_mask_op op)
  404. {
  405. bitmap_super_t *sb;
  406. unsigned long flags;
  407. spin_lock_irqsave(&bitmap->lock, flags);
  408. if (!bitmap || !bitmap->sb_page) { /* can't set the state */
  409. spin_unlock_irqrestore(&bitmap->lock, flags);
  410. return;
  411. }
  412. page_cache_get(bitmap->sb_page);
  413. spin_unlock_irqrestore(&bitmap->lock, flags);
  414. sb = (bitmap_super_t *)kmap(bitmap->sb_page);
  415. switch (op) {
  416. case MASK_SET: sb->state |= bits;
  417. break;
  418. case MASK_UNSET: sb->state &= ~bits;
  419. break;
  420. default: BUG();
  421. }
  422. kunmap(bitmap->sb_page);
  423. page_cache_release(bitmap->sb_page);
  424. }
  425. /*
  426. * general bitmap file operations
  427. */
  428. /* calculate the index of the page that contains this bit */
  429. static inline unsigned long file_page_index(unsigned long chunk)
  430. {
  431. return CHUNK_BIT_OFFSET(chunk) >> PAGE_BIT_SHIFT;
  432. }
  433. /* calculate the (bit) offset of this bit within a page */
  434. static inline unsigned long file_page_offset(unsigned long chunk)
  435. {
  436. return CHUNK_BIT_OFFSET(chunk) & (PAGE_BITS - 1);
  437. }
  438. /*
  439. * return a pointer to the page in the filemap that contains the given bit
  440. *
  441. * this lookup is complicated by the fact that the bitmap sb might be exactly
  442. * 1 page (e.g., x86) or less than 1 page -- so the bitmap might start on page
  443. * 0 or page 1
  444. */
  445. static inline struct page *filemap_get_page(struct bitmap *bitmap,
  446. unsigned long chunk)
  447. {
  448. return bitmap->filemap[file_page_index(chunk) - file_page_index(0)];
  449. }
  450. static void bitmap_file_unmap(struct bitmap *bitmap)
  451. {
  452. struct page **map, *sb_page;
  453. unsigned long *attr;
  454. int pages;
  455. unsigned long flags;
  456. spin_lock_irqsave(&bitmap->lock, flags);
  457. map = bitmap->filemap;
  458. bitmap->filemap = NULL;
  459. attr = bitmap->filemap_attr;
  460. bitmap->filemap_attr = NULL;
  461. pages = bitmap->file_pages;
  462. bitmap->file_pages = 0;
  463. sb_page = bitmap->sb_page;
  464. bitmap->sb_page = NULL;
  465. spin_unlock_irqrestore(&bitmap->lock, flags);
  466. while (pages--)
  467. if (map[pages]->index != 0) /* 0 is sb_page, release it below */
  468. page_cache_release(map[pages]);
  469. kfree(map);
  470. kfree(attr);
  471. if (sb_page)
  472. page_cache_release(sb_page);
  473. }
  474. static void bitmap_stop_daemons(struct bitmap *bitmap);
  475. /* dequeue the next item in a page list -- don't call from irq context */
  476. static struct page_list *dequeue_page(struct bitmap *bitmap)
  477. {
  478. struct page_list *item = NULL;
  479. struct list_head *head = &bitmap->complete_pages;
  480. spin_lock(&bitmap->write_lock);
  481. if (list_empty(head))
  482. goto out;
  483. item = list_entry(head->prev, struct page_list, list);
  484. list_del(head->prev);
  485. out:
  486. spin_unlock(&bitmap->write_lock);
  487. return item;
  488. }
  489. static void drain_write_queues(struct bitmap *bitmap)
  490. {
  491. struct page_list *item;
  492. while ((item = dequeue_page(bitmap))) {
  493. /* don't bother to wait */
  494. page_cache_release(item->page);
  495. mempool_free(item, bitmap->write_pool);
  496. }
  497. wake_up(&bitmap->write_wait);
  498. }
  499. static void bitmap_file_put(struct bitmap *bitmap)
  500. {
  501. struct file *file;
  502. struct inode *inode;
  503. unsigned long flags;
  504. spin_lock_irqsave(&bitmap->lock, flags);
  505. file = bitmap->file;
  506. bitmap->file = NULL;
  507. spin_unlock_irqrestore(&bitmap->lock, flags);
  508. bitmap_stop_daemons(bitmap);
  509. drain_write_queues(bitmap);
  510. bitmap_file_unmap(bitmap);
  511. if (file) {
  512. inode = file->f_mapping->host;
  513. spin_lock(&inode->i_lock);
  514. atomic_set(&inode->i_writecount, 1); /* allow writes again */
  515. spin_unlock(&inode->i_lock);
  516. fput(file);
  517. }
  518. }
  519. /*
  520. * bitmap_file_kick - if an error occurs while manipulating the bitmap file
  521. * then it is no longer reliable, so we stop using it and we mark the file
  522. * as failed in the superblock
  523. */
  524. static void bitmap_file_kick(struct bitmap *bitmap)
  525. {
  526. char *path, *ptr = NULL;
  527. bitmap_mask_state(bitmap, BITMAP_STALE, MASK_SET);
  528. bitmap_update_sb(bitmap);
  529. path = kmalloc(PAGE_SIZE, GFP_KERNEL);
  530. if (path)
  531. ptr = file_path(bitmap->file, path, PAGE_SIZE);
  532. printk(KERN_ALERT "%s: kicking failed bitmap file %s from array!\n",
  533. bmname(bitmap), ptr ? ptr : "");
  534. kfree(path);
  535. bitmap_file_put(bitmap);
  536. return;
  537. }
  538. enum bitmap_page_attr {
  539. BITMAP_PAGE_DIRTY = 1, // there are set bits that need to be synced
  540. BITMAP_PAGE_CLEAN = 2, // there are bits that might need to be cleared
  541. BITMAP_PAGE_NEEDWRITE=4, // there are cleared bits that need to be synced
  542. };
  543. static inline void set_page_attr(struct bitmap *bitmap, struct page *page,
  544. enum bitmap_page_attr attr)
  545. {
  546. bitmap->filemap_attr[page->index] |= attr;
  547. }
  548. static inline void clear_page_attr(struct bitmap *bitmap, struct page *page,
  549. enum bitmap_page_attr attr)
  550. {
  551. bitmap->filemap_attr[page->index] &= ~attr;
  552. }
  553. static inline unsigned long get_page_attr(struct bitmap *bitmap, struct page *page)
  554. {
  555. return bitmap->filemap_attr[page->index];
  556. }
  557. /*
  558. * bitmap_file_set_bit -- called before performing a write to the md device
  559. * to set (and eventually sync) a particular bit in the bitmap file
  560. *
  561. * we set the bit immediately, then we record the page number so that
  562. * when an unplug occurs, we can flush the dirty pages out to disk
  563. */
  564. static void bitmap_file_set_bit(struct bitmap *bitmap, sector_t block)
  565. {
  566. unsigned long bit;
  567. struct page *page;
  568. void *kaddr;
  569. unsigned long chunk = block >> CHUNK_BLOCK_SHIFT(bitmap);
  570. if (!bitmap->file || !bitmap->filemap) {
  571. return;
  572. }
  573. page = filemap_get_page(bitmap, chunk);
  574. bit = file_page_offset(chunk);
  575. /* make sure the page stays cached until it gets written out */
  576. if (! (get_page_attr(bitmap, page) & BITMAP_PAGE_DIRTY))
  577. page_cache_get(page);
  578. /* set the bit */
  579. kaddr = kmap_atomic(page, KM_USER0);
  580. set_bit(bit, kaddr);
  581. kunmap_atomic(kaddr, KM_USER0);
  582. PRINTK("set file bit %lu page %lu\n", bit, page->index);
  583. /* record page number so it gets flushed to disk when unplug occurs */
  584. set_page_attr(bitmap, page, BITMAP_PAGE_DIRTY);
  585. }
  586. /* this gets called when the md device is ready to unplug its underlying
  587. * (slave) device queues -- before we let any writes go down, we need to
  588. * sync the dirty pages of the bitmap file to disk */
  589. int bitmap_unplug(struct bitmap *bitmap)
  590. {
  591. unsigned long i, attr, flags;
  592. struct page *page;
  593. int wait = 0;
  594. if (!bitmap)
  595. return 0;
  596. /* look at each page to see if there are any set bits that need to be
  597. * flushed out to disk */
  598. for (i = 0; i < bitmap->file_pages; i++) {
  599. spin_lock_irqsave(&bitmap->lock, flags);
  600. if (!bitmap->file || !bitmap->filemap) {
  601. spin_unlock_irqrestore(&bitmap->lock, flags);
  602. return 0;
  603. }
  604. page = bitmap->filemap[i];
  605. attr = get_page_attr(bitmap, page);
  606. clear_page_attr(bitmap, page, BITMAP_PAGE_DIRTY);
  607. clear_page_attr(bitmap, page, BITMAP_PAGE_NEEDWRITE);
  608. if ((attr & BITMAP_PAGE_DIRTY))
  609. wait = 1;
  610. spin_unlock_irqrestore(&bitmap->lock, flags);
  611. if (attr & (BITMAP_PAGE_DIRTY | BITMAP_PAGE_NEEDWRITE))
  612. if (write_page(bitmap, page, 0))
  613. return 1;
  614. }
  615. if (wait) { /* if any writes were performed, we need to wait on them */
  616. spin_lock_irq(&bitmap->write_lock);
  617. wait_event_lock_irq(bitmap->write_wait,
  618. list_empty(&bitmap->complete_pages), bitmap->write_lock,
  619. wake_up_process(bitmap->writeback_daemon->tsk));
  620. spin_unlock_irq(&bitmap->write_lock);
  621. }
  622. return 0;
  623. }
  624. static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset,
  625. unsigned long sectors, int in_sync);
  626. /* * bitmap_init_from_disk -- called at bitmap_create time to initialize
  627. * the in-memory bitmap from the on-disk bitmap -- also, sets up the
  628. * memory mapping of the bitmap file
  629. * Special cases:
  630. * if there's no bitmap file, or if the bitmap file had been
  631. * previously kicked from the array, we mark all the bits as
  632. * 1's in order to cause a full resync.
  633. */
  634. static int bitmap_init_from_disk(struct bitmap *bitmap, int in_sync)
  635. {
  636. unsigned long i, chunks, index, oldindex, bit;
  637. struct page *page = NULL, *oldpage = NULL;
  638. unsigned long num_pages, bit_cnt = 0;
  639. struct file *file;
  640. unsigned long bytes, offset, dummy;
  641. int outofdate;
  642. int ret = -ENOSPC;
  643. chunks = bitmap->chunks;
  644. file = bitmap->file;
  645. BUG_ON(!file);
  646. #if INJECT_FAULTS_3
  647. outofdate = 1;
  648. #else
  649. outofdate = bitmap->flags & BITMAP_STALE;
  650. #endif
  651. if (outofdate)
  652. printk(KERN_INFO "%s: bitmap file is out of date, doing full "
  653. "recovery\n", bmname(bitmap));
  654. bytes = (chunks + 7) / 8;
  655. num_pages = (bytes + sizeof(bitmap_super_t) + PAGE_SIZE - 1) / PAGE_SIZE;
  656. if (i_size_read(file->f_mapping->host) < bytes + sizeof(bitmap_super_t)) {
  657. printk(KERN_INFO "%s: bitmap file too short %lu < %lu\n",
  658. bmname(bitmap),
  659. (unsigned long) i_size_read(file->f_mapping->host),
  660. bytes + sizeof(bitmap_super_t));
  661. goto out;
  662. }
  663. ret = -ENOMEM;
  664. bitmap->filemap = kmalloc(sizeof(struct page *) * num_pages, GFP_KERNEL);
  665. if (!bitmap->filemap)
  666. goto out;
  667. bitmap->filemap_attr = kmalloc(sizeof(long) * num_pages, GFP_KERNEL);
  668. if (!bitmap->filemap_attr)
  669. goto out;
  670. memset(bitmap->filemap_attr, 0, sizeof(long) * num_pages);
  671. oldindex = ~0L;
  672. for (i = 0; i < chunks; i++) {
  673. index = file_page_index(i);
  674. bit = file_page_offset(i);
  675. if (index != oldindex) { /* this is a new page, read it in */
  676. /* unmap the old page, we're done with it */
  677. if (oldpage != NULL)
  678. kunmap(oldpage);
  679. if (index == 0) {
  680. /*
  681. * if we're here then the superblock page
  682. * contains some bits (PAGE_SIZE != sizeof sb)
  683. * we've already read it in, so just use it
  684. */
  685. page = bitmap->sb_page;
  686. offset = sizeof(bitmap_super_t);
  687. } else {
  688. page = read_page(file, index, &dummy);
  689. if (IS_ERR(page)) { /* read error */
  690. ret = PTR_ERR(page);
  691. goto out;
  692. }
  693. offset = 0;
  694. }
  695. oldindex = index;
  696. oldpage = page;
  697. kmap(page);
  698. if (outofdate) {
  699. /*
  700. * if bitmap is out of date, dirty the
  701. * whole page and write it out
  702. */
  703. memset(page_address(page) + offset, 0xff,
  704. PAGE_SIZE - offset);
  705. ret = write_page(bitmap, page, 1);
  706. if (ret) {
  707. kunmap(page);
  708. /* release, page not in filemap yet */
  709. page_cache_release(page);
  710. goto out;
  711. }
  712. }
  713. bitmap->filemap[bitmap->file_pages++] = page;
  714. }
  715. if (test_bit(bit, page_address(page))) {
  716. /* if the disk bit is set, set the memory bit */
  717. bitmap_set_memory_bits(bitmap,
  718. i << CHUNK_BLOCK_SHIFT(bitmap), 1, in_sync);
  719. bit_cnt++;
  720. }
  721. }
  722. /* everything went OK */
  723. ret = 0;
  724. bitmap_mask_state(bitmap, BITMAP_STALE, MASK_UNSET);
  725. if (page) /* unmap the last page */
  726. kunmap(page);
  727. if (bit_cnt) { /* Kick recovery if any bits were set */
  728. set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery);
  729. md_wakeup_thread(bitmap->mddev->thread);
  730. }
  731. out:
  732. printk(KERN_INFO "%s: bitmap initialized from disk: "
  733. "read %lu/%lu pages, set %lu bits, status: %d\n",
  734. bmname(bitmap), bitmap->file_pages, num_pages, bit_cnt, ret);
  735. return ret;
  736. }
  737. static void bitmap_count_page(struct bitmap *bitmap, sector_t offset, int inc)
  738. {
  739. sector_t chunk = offset >> CHUNK_BLOCK_SHIFT(bitmap);
  740. unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
  741. bitmap->bp[page].count += inc;
  742. /*
  743. if (page == 0) printk("count page 0, offset %llu: %d gives %d\n",
  744. (unsigned long long)offset, inc, bitmap->bp[page].count);
  745. */
  746. bitmap_checkfree(bitmap, page);
  747. }
  748. static bitmap_counter_t *bitmap_get_counter(struct bitmap *bitmap,
  749. sector_t offset, int *blocks,
  750. int create);
  751. /*
  752. * bitmap daemon -- periodically wakes up to clean bits and flush pages
  753. * out to disk
  754. */
  755. int bitmap_daemon_work(struct bitmap *bitmap)
  756. {
  757. unsigned long j;
  758. unsigned long flags;
  759. struct page *page = NULL, *lastpage = NULL;
  760. int err = 0;
  761. int blocks;
  762. int attr;
  763. if (bitmap == NULL)
  764. return 0;
  765. if (time_before(jiffies, bitmap->daemon_lastrun + bitmap->daemon_sleep*HZ))
  766. return 0;
  767. bitmap->daemon_lastrun = jiffies;
  768. for (j = 0; j < bitmap->chunks; j++) {
  769. bitmap_counter_t *bmc;
  770. spin_lock_irqsave(&bitmap->lock, flags);
  771. if (!bitmap->file || !bitmap->filemap) {
  772. /* error or shutdown */
  773. spin_unlock_irqrestore(&bitmap->lock, flags);
  774. break;
  775. }
  776. page = filemap_get_page(bitmap, j);
  777. if (page != lastpage) {
  778. /* skip this page unless it's marked as needing cleaning */
  779. if (!((attr=get_page_attr(bitmap, page)) & BITMAP_PAGE_CLEAN)) {
  780. if (attr & BITMAP_PAGE_NEEDWRITE) {
  781. page_cache_get(page);
  782. clear_page_attr(bitmap, page, BITMAP_PAGE_NEEDWRITE);
  783. }
  784. spin_unlock_irqrestore(&bitmap->lock, flags);
  785. if (attr & BITMAP_PAGE_NEEDWRITE) {
  786. if (write_page(bitmap, page, 0))
  787. bitmap_file_kick(bitmap);
  788. page_cache_release(page);
  789. }
  790. continue;
  791. }
  792. /* grab the new page, sync and release the old */
  793. page_cache_get(page);
  794. if (lastpage != NULL) {
  795. if (get_page_attr(bitmap, lastpage) & BITMAP_PAGE_NEEDWRITE) {
  796. clear_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE);
  797. spin_unlock_irqrestore(&bitmap->lock, flags);
  798. err = write_page(bitmap, lastpage, 0);
  799. } else {
  800. set_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE);
  801. spin_unlock_irqrestore(&bitmap->lock, flags);
  802. }
  803. kunmap(lastpage);
  804. page_cache_release(lastpage);
  805. if (err)
  806. bitmap_file_kick(bitmap);
  807. } else
  808. spin_unlock_irqrestore(&bitmap->lock, flags);
  809. lastpage = page;
  810. kmap(page);
  811. /*
  812. printk("bitmap clean at page %lu\n", j);
  813. */
  814. spin_lock_irqsave(&bitmap->lock, flags);
  815. clear_page_attr(bitmap, page, BITMAP_PAGE_CLEAN);
  816. }
  817. bmc = bitmap_get_counter(bitmap, j << CHUNK_BLOCK_SHIFT(bitmap),
  818. &blocks, 0);
  819. if (bmc) {
  820. /*
  821. if (j < 100) printk("bitmap: j=%lu, *bmc = 0x%x\n", j, *bmc);
  822. */
  823. if (*bmc == 2) {
  824. *bmc=1; /* maybe clear the bit next time */
  825. set_page_attr(bitmap, page, BITMAP_PAGE_CLEAN);
  826. } else if (*bmc == 1) {
  827. /* we can clear the bit */
  828. *bmc = 0;
  829. bitmap_count_page(bitmap, j << CHUNK_BLOCK_SHIFT(bitmap),
  830. -1);
  831. /* clear the bit */
  832. clear_bit(file_page_offset(j), page_address(page));
  833. }
  834. }
  835. spin_unlock_irqrestore(&bitmap->lock, flags);
  836. }
  837. /* now sync the final page */
  838. if (lastpage != NULL) {
  839. kunmap(lastpage);
  840. spin_lock_irqsave(&bitmap->lock, flags);
  841. if (get_page_attr(bitmap, lastpage) &BITMAP_PAGE_NEEDWRITE) {
  842. clear_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE);
  843. spin_unlock_irqrestore(&bitmap->lock, flags);
  844. err = write_page(bitmap, lastpage, 0);
  845. } else {
  846. set_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE);
  847. spin_unlock_irqrestore(&bitmap->lock, flags);
  848. }
  849. page_cache_release(lastpage);
  850. }
  851. return err;
  852. }
  853. static void daemon_exit(struct bitmap *bitmap, mdk_thread_t **daemon)
  854. {
  855. mdk_thread_t *dmn;
  856. unsigned long flags;
  857. /* if no one is waiting on us, we'll free the md thread struct
  858. * and exit, otherwise we let the waiter clean things up */
  859. spin_lock_irqsave(&bitmap->lock, flags);
  860. if ((dmn = *daemon)) { /* no one is waiting, cleanup and exit */
  861. *daemon = NULL;
  862. spin_unlock_irqrestore(&bitmap->lock, flags);
  863. kfree(dmn);
  864. complete_and_exit(NULL, 0); /* do_exit not exported */
  865. }
  866. spin_unlock_irqrestore(&bitmap->lock, flags);
  867. }
  868. static void bitmap_writeback_daemon(mddev_t *mddev)
  869. {
  870. struct bitmap *bitmap = mddev->bitmap;
  871. struct page *page;
  872. struct page_list *item;
  873. int err = 0;
  874. if (signal_pending(current)) {
  875. printk(KERN_INFO
  876. "%s: bitmap writeback daemon got signal, exiting...\n",
  877. bmname(bitmap));
  878. err = -EINTR;
  879. goto out;
  880. }
  881. PRINTK("%s: bitmap writeback daemon woke up...\n", bmname(bitmap));
  882. /* wait on bitmap page writebacks */
  883. while ((item = dequeue_page(bitmap))) {
  884. page = item->page;
  885. mempool_free(item, bitmap->write_pool);
  886. PRINTK("wait on page writeback: %p\n", page);
  887. wait_on_page_writeback(page);
  888. PRINTK("finished page writeback: %p\n", page);
  889. err = PageError(page);
  890. page_cache_release(page);
  891. if (err) {
  892. printk(KERN_WARNING "%s: bitmap file writeback "
  893. "failed (page %lu): %d\n",
  894. bmname(bitmap), page->index, err);
  895. bitmap_file_kick(bitmap);
  896. goto out;
  897. }
  898. }
  899. out:
  900. wake_up(&bitmap->write_wait);
  901. if (err) {
  902. printk(KERN_INFO "%s: bitmap writeback daemon exiting (%d)\n",
  903. bmname(bitmap), err);
  904. daemon_exit(bitmap, &bitmap->writeback_daemon);
  905. }
  906. }
  907. static int bitmap_start_daemon(struct bitmap *bitmap, mdk_thread_t **ptr,
  908. void (*func)(mddev_t *), char *name)
  909. {
  910. mdk_thread_t *daemon;
  911. unsigned long flags;
  912. char namebuf[32];
  913. spin_lock_irqsave(&bitmap->lock, flags);
  914. *ptr = NULL;
  915. if (!bitmap->file) /* no need for daemon if there's no backing file */
  916. goto out_unlock;
  917. spin_unlock_irqrestore(&bitmap->lock, flags);
  918. #if INJECT_FATAL_FAULT_2
  919. daemon = NULL;
  920. #else
  921. sprintf(namebuf, "%%s_%s", name);
  922. daemon = md_register_thread(func, bitmap->mddev, namebuf);
  923. #endif
  924. if (!daemon) {
  925. printk(KERN_ERR "%s: failed to start bitmap daemon\n",
  926. bmname(bitmap));
  927. return -ECHILD;
  928. }
  929. spin_lock_irqsave(&bitmap->lock, flags);
  930. *ptr = daemon;
  931. md_wakeup_thread(daemon); /* start it running */
  932. PRINTK("%s: %s daemon (pid %d) started...\n",
  933. bmname(bitmap), name, daemon->tsk->pid);
  934. out_unlock:
  935. spin_unlock_irqrestore(&bitmap->lock, flags);
  936. return 0;
  937. }
  938. static int bitmap_start_daemons(struct bitmap *bitmap)
  939. {
  940. int err = bitmap_start_daemon(bitmap, &bitmap->writeback_daemon,
  941. bitmap_writeback_daemon, "bitmap_wb");
  942. return err;
  943. }
  944. static void bitmap_stop_daemon(struct bitmap *bitmap, mdk_thread_t **ptr)
  945. {
  946. mdk_thread_t *daemon;
  947. unsigned long flags;
  948. spin_lock_irqsave(&bitmap->lock, flags);
  949. daemon = *ptr;
  950. *ptr = NULL;
  951. spin_unlock_irqrestore(&bitmap->lock, flags);
  952. if (daemon)
  953. md_unregister_thread(daemon); /* destroy the thread */
  954. }
  955. static void bitmap_stop_daemons(struct bitmap *bitmap)
  956. {
  957. /* the daemons can't stop themselves... they'll just exit instead... */
  958. if (bitmap->writeback_daemon &&
  959. current->pid != bitmap->writeback_daemon->tsk->pid)
  960. bitmap_stop_daemon(bitmap, &bitmap->writeback_daemon);
  961. }
  962. static bitmap_counter_t *bitmap_get_counter(struct bitmap *bitmap,
  963. sector_t offset, int *blocks,
  964. int create)
  965. {
  966. /* If 'create', we might release the lock and reclaim it.
  967. * The lock must have been taken with interrupts enabled.
  968. * If !create, we don't release the lock.
  969. */
  970. sector_t chunk = offset >> CHUNK_BLOCK_SHIFT(bitmap);
  971. unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
  972. unsigned long pageoff = (chunk & PAGE_COUNTER_MASK) << COUNTER_BYTE_SHIFT;
  973. sector_t csize;
  974. if (bitmap_checkpage(bitmap, page, create) < 0) {
  975. csize = ((sector_t)1) << (CHUNK_BLOCK_SHIFT(bitmap));
  976. *blocks = csize - (offset & (csize- 1));
  977. return NULL;
  978. }
  979. /* now locked ... */
  980. if (bitmap->bp[page].hijacked) { /* hijacked pointer */
  981. /* should we use the first or second counter field
  982. * of the hijacked pointer? */
  983. int hi = (pageoff > PAGE_COUNTER_MASK);
  984. csize = ((sector_t)1) << (CHUNK_BLOCK_SHIFT(bitmap) +
  985. PAGE_COUNTER_SHIFT - 1);
  986. *blocks = csize - (offset & (csize- 1));
  987. return &((bitmap_counter_t *)
  988. &bitmap->bp[page].map)[hi];
  989. } else { /* page is allocated */
  990. csize = ((sector_t)1) << (CHUNK_BLOCK_SHIFT(bitmap));
  991. *blocks = csize - (offset & (csize- 1));
  992. return (bitmap_counter_t *)
  993. &(bitmap->bp[page].map[pageoff]);
  994. }
  995. }
  996. int bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors)
  997. {
  998. if (!bitmap) return 0;
  999. while (sectors) {
  1000. int blocks;
  1001. bitmap_counter_t *bmc;
  1002. spin_lock_irq(&bitmap->lock);
  1003. bmc = bitmap_get_counter(bitmap, offset, &blocks, 1);
  1004. if (!bmc) {
  1005. spin_unlock_irq(&bitmap->lock);
  1006. return 0;
  1007. }
  1008. switch(*bmc) {
  1009. case 0:
  1010. bitmap_file_set_bit(bitmap, offset);
  1011. bitmap_count_page(bitmap,offset, 1);
  1012. blk_plug_device(bitmap->mddev->queue);
  1013. /* fall through */
  1014. case 1:
  1015. *bmc = 2;
  1016. }
  1017. if ((*bmc & COUNTER_MAX) == COUNTER_MAX) BUG();
  1018. (*bmc)++;
  1019. spin_unlock_irq(&bitmap->lock);
  1020. offset += blocks;
  1021. if (sectors > blocks)
  1022. sectors -= blocks;
  1023. else sectors = 0;
  1024. }
  1025. return 0;
  1026. }
  1027. void bitmap_endwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors,
  1028. int success)
  1029. {
  1030. if (!bitmap) return;
  1031. while (sectors) {
  1032. int blocks;
  1033. unsigned long flags;
  1034. bitmap_counter_t *bmc;
  1035. spin_lock_irqsave(&bitmap->lock, flags);
  1036. bmc = bitmap_get_counter(bitmap, offset, &blocks, 0);
  1037. if (!bmc) {
  1038. spin_unlock_irqrestore(&bitmap->lock, flags);
  1039. return;
  1040. }
  1041. if (!success && ! (*bmc & NEEDED_MASK))
  1042. *bmc |= NEEDED_MASK;
  1043. (*bmc)--;
  1044. if (*bmc <= 2) {
  1045. set_page_attr(bitmap,
  1046. filemap_get_page(bitmap, offset >> CHUNK_BLOCK_SHIFT(bitmap)),
  1047. BITMAP_PAGE_CLEAN);
  1048. }
  1049. spin_unlock_irqrestore(&bitmap->lock, flags);
  1050. offset += blocks;
  1051. if (sectors > blocks)
  1052. sectors -= blocks;
  1053. else sectors = 0;
  1054. }
  1055. }
  1056. int bitmap_start_sync(struct bitmap *bitmap, sector_t offset, int *blocks)
  1057. {
  1058. bitmap_counter_t *bmc;
  1059. int rv;
  1060. if (bitmap == NULL) {/* FIXME or bitmap set as 'failed' */
  1061. *blocks = 1024;
  1062. return 1; /* always resync if no bitmap */
  1063. }
  1064. spin_lock_irq(&bitmap->lock);
  1065. bmc = bitmap_get_counter(bitmap, offset, blocks, 0);
  1066. rv = 0;
  1067. if (bmc) {
  1068. /* locked */
  1069. if (RESYNC(*bmc))
  1070. rv = 1;
  1071. else if (NEEDED(*bmc)) {
  1072. rv = 1;
  1073. *bmc |= RESYNC_MASK;
  1074. *bmc &= ~NEEDED_MASK;
  1075. }
  1076. }
  1077. spin_unlock_irq(&bitmap->lock);
  1078. return rv;
  1079. }
  1080. void bitmap_end_sync(struct bitmap *bitmap, sector_t offset, int *blocks, int aborted)
  1081. {
  1082. bitmap_counter_t *bmc;
  1083. unsigned long flags;
  1084. /*
  1085. if (offset == 0) printk("bitmap_end_sync 0 (%d)\n", aborted);
  1086. */ if (bitmap == NULL) {
  1087. *blocks = 1024;
  1088. return;
  1089. }
  1090. spin_lock_irqsave(&bitmap->lock, flags);
  1091. bmc = bitmap_get_counter(bitmap, offset, blocks, 0);
  1092. if (bmc == NULL)
  1093. goto unlock;
  1094. /* locked */
  1095. /*
  1096. if (offset == 0) printk("bitmap_end sync found 0x%x, blocks %d\n", *bmc, *blocks);
  1097. */
  1098. if (RESYNC(*bmc)) {
  1099. *bmc &= ~RESYNC_MASK;
  1100. if (!NEEDED(*bmc) && aborted)
  1101. *bmc |= NEEDED_MASK;
  1102. else {
  1103. if (*bmc <= 2) {
  1104. set_page_attr(bitmap,
  1105. filemap_get_page(bitmap, offset >> CHUNK_BLOCK_SHIFT(bitmap)),
  1106. BITMAP_PAGE_CLEAN);
  1107. }
  1108. }
  1109. }
  1110. unlock:
  1111. spin_unlock_irqrestore(&bitmap->lock, flags);
  1112. }
  1113. void bitmap_close_sync(struct bitmap *bitmap)
  1114. {
  1115. /* Sync has finished, and any bitmap chunks that weren't synced
  1116. * properly have been aborted. It remains to us to clear the
  1117. * RESYNC bit wherever it is still on
  1118. */
  1119. sector_t sector = 0;
  1120. int blocks;
  1121. if (!bitmap) return;
  1122. while (sector < bitmap->mddev->resync_max_sectors) {
  1123. bitmap_end_sync(bitmap, sector, &blocks, 0);
  1124. /*
  1125. if (sector < 500) printk("bitmap_close_sync: sec %llu blks %d\n",
  1126. (unsigned long long)sector, blocks);
  1127. */ sector += blocks;
  1128. }
  1129. }
  1130. static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset,
  1131. unsigned long sectors, int in_sync)
  1132. {
  1133. /* For each chunk covered by any of these sectors, set the
  1134. * counter to 1 and set resync_needed unless in_sync. They should all
  1135. * be 0 at this point
  1136. */
  1137. while (sectors) {
  1138. int secs;
  1139. bitmap_counter_t *bmc;
  1140. spin_lock_irq(&bitmap->lock);
  1141. bmc = bitmap_get_counter(bitmap, offset, &secs, 1);
  1142. if (!bmc) {
  1143. spin_unlock_irq(&bitmap->lock);
  1144. return;
  1145. }
  1146. if (! *bmc) {
  1147. struct page *page;
  1148. *bmc = 1 | (in_sync? 0 : NEEDED_MASK);
  1149. bitmap_count_page(bitmap, offset, 1);
  1150. page = filemap_get_page(bitmap, offset >> CHUNK_BLOCK_SHIFT(bitmap));
  1151. set_page_attr(bitmap, page, BITMAP_PAGE_CLEAN);
  1152. }
  1153. spin_unlock_irq(&bitmap->lock);
  1154. if (sectors > secs)
  1155. sectors -= secs;
  1156. else
  1157. sectors = 0;
  1158. }
  1159. }
  1160. /* dirty the entire bitmap */
  1161. int bitmap_setallbits(struct bitmap *bitmap)
  1162. {
  1163. unsigned long flags;
  1164. unsigned long j;
  1165. /* dirty the in-memory bitmap */
  1166. bitmap_set_memory_bits(bitmap, 0, bitmap->chunks << CHUNK_BLOCK_SHIFT(bitmap), 1);
  1167. /* dirty the bitmap file */
  1168. for (j = 0; j < bitmap->file_pages; j++) {
  1169. struct page *page = bitmap->filemap[j];
  1170. spin_lock_irqsave(&bitmap->lock, flags);
  1171. page_cache_get(page);
  1172. spin_unlock_irqrestore(&bitmap->lock, flags);
  1173. memset(kmap(page), 0xff, PAGE_SIZE);
  1174. kunmap(page);
  1175. if (write_page(bitmap, page, 0))
  1176. return 1;
  1177. }
  1178. return 0;
  1179. }
  1180. /*
  1181. * free memory that was allocated
  1182. */
  1183. void bitmap_destroy(mddev_t *mddev)
  1184. {
  1185. unsigned long k, pages;
  1186. struct bitmap_page *bp;
  1187. struct bitmap *bitmap = mddev->bitmap;
  1188. if (!bitmap) /* there was no bitmap */
  1189. return;
  1190. mddev->bitmap = NULL; /* disconnect from the md device */
  1191. /* release the bitmap file and kill the daemon */
  1192. bitmap_file_put(bitmap);
  1193. bp = bitmap->bp;
  1194. pages = bitmap->pages;
  1195. /* free all allocated memory */
  1196. mempool_destroy(bitmap->write_pool);
  1197. if (bp) /* deallocate the page memory */
  1198. for (k = 0; k < pages; k++)
  1199. if (bp[k].map && !bp[k].hijacked)
  1200. kfree(bp[k].map);
  1201. kfree(bp);
  1202. kfree(bitmap);
  1203. }
  1204. /*
  1205. * initialize the bitmap structure
  1206. * if this returns an error, bitmap_destroy must be called to do clean up
  1207. */
  1208. int bitmap_create(mddev_t *mddev)
  1209. {
  1210. struct bitmap *bitmap;
  1211. unsigned long blocks = mddev->resync_max_sectors;
  1212. unsigned long chunks;
  1213. unsigned long pages;
  1214. struct file *file = mddev->bitmap_file;
  1215. int err;
  1216. BUG_ON(sizeof(bitmap_super_t) != 256);
  1217. if (!file) /* bitmap disabled, nothing to do */
  1218. return 0;
  1219. bitmap = kmalloc(sizeof(*bitmap), GFP_KERNEL);
  1220. if (!bitmap)
  1221. return -ENOMEM;
  1222. memset(bitmap, 0, sizeof(*bitmap));
  1223. spin_lock_init(&bitmap->lock);
  1224. bitmap->mddev = mddev;
  1225. mddev->bitmap = bitmap;
  1226. spin_lock_init(&bitmap->write_lock);
  1227. INIT_LIST_HEAD(&bitmap->complete_pages);
  1228. init_waitqueue_head(&bitmap->write_wait);
  1229. bitmap->write_pool = mempool_create(WRITE_POOL_SIZE, write_pool_alloc,
  1230. write_pool_free, NULL);
  1231. if (!bitmap->write_pool)
  1232. return -ENOMEM;
  1233. bitmap->file = file;
  1234. get_file(file);
  1235. /* read superblock from bitmap file (this sets bitmap->chunksize) */
  1236. err = bitmap_read_sb(bitmap);
  1237. if (err)
  1238. return err;
  1239. bitmap->chunkshift = find_first_bit(&bitmap->chunksize,
  1240. sizeof(bitmap->chunksize));
  1241. /* now that chunksize and chunkshift are set, we can use these macros */
  1242. chunks = (blocks + CHUNK_BLOCK_RATIO(bitmap) - 1) /
  1243. CHUNK_BLOCK_RATIO(bitmap);
  1244. pages = (chunks + PAGE_COUNTER_RATIO - 1) / PAGE_COUNTER_RATIO;
  1245. BUG_ON(!pages);
  1246. bitmap->chunks = chunks;
  1247. bitmap->pages = pages;
  1248. bitmap->missing_pages = pages;
  1249. bitmap->counter_bits = COUNTER_BITS;
  1250. bitmap->syncchunk = ~0UL;
  1251. #if INJECT_FATAL_FAULT_1
  1252. bitmap->bp = NULL;
  1253. #else
  1254. bitmap->bp = kmalloc(pages * sizeof(*bitmap->bp), GFP_KERNEL);
  1255. #endif
  1256. if (!bitmap->bp)
  1257. return -ENOMEM;
  1258. memset(bitmap->bp, 0, pages * sizeof(*bitmap->bp));
  1259. bitmap->flags |= BITMAP_ACTIVE;
  1260. /* now that we have some pages available, initialize the in-memory
  1261. * bitmap from the on-disk bitmap */
  1262. err = bitmap_init_from_disk(bitmap, mddev->recovery_cp == MaxSector);
  1263. if (err)
  1264. return err;
  1265. printk(KERN_INFO "created bitmap (%lu pages) for device %s\n",
  1266. pages, bmname(bitmap));
  1267. /* kick off the bitmap daemons */
  1268. err = bitmap_start_daemons(bitmap);
  1269. if (err)
  1270. return err;
  1271. return bitmap_update_sb(bitmap);
  1272. }
  1273. /* the bitmap API -- for raid personalities */
  1274. EXPORT_SYMBOL(bitmap_startwrite);
  1275. EXPORT_SYMBOL(bitmap_endwrite);
  1276. EXPORT_SYMBOL(bitmap_start_sync);
  1277. EXPORT_SYMBOL(bitmap_end_sync);
  1278. EXPORT_SYMBOL(bitmap_unplug);
  1279. EXPORT_SYMBOL(bitmap_close_sync);
  1280. EXPORT_SYMBOL(bitmap_daemon_work);