xfs_buf.c 40 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840
  1. /*
  2. * Copyright (c) 2000-2006 Silicon Graphics, Inc.
  3. * All Rights Reserved.
  4. *
  5. * This program is free software; you can redistribute it and/or
  6. * modify it under the terms of the GNU General Public License as
  7. * published by the Free Software Foundation.
  8. *
  9. * This program is distributed in the hope that it would be useful,
  10. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. * GNU General Public License for more details.
  13. *
  14. * You should have received a copy of the GNU General Public License
  15. * along with this program; if not, write the Free Software Foundation,
  16. * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  17. */
  18. #include "xfs.h"
  19. #include <linux/stddef.h>
  20. #include <linux/errno.h>
  21. #include <linux/slab.h>
  22. #include <linux/pagemap.h>
  23. #include <linux/init.h>
  24. #include <linux/vmalloc.h>
  25. #include <linux/bio.h>
  26. #include <linux/sysctl.h>
  27. #include <linux/proc_fs.h>
  28. #include <linux/workqueue.h>
  29. #include <linux/percpu.h>
  30. #include <linux/blkdev.h>
  31. #include <linux/hash.h>
  32. #include <linux/kthread.h>
  33. #include <linux/migrate.h>
  34. #include <linux/backing-dev.h>
  35. #include <linux/freezer.h>
  36. #include "xfs_sb.h"
  37. #include "xfs_inum.h"
  38. #include "xfs_ag.h"
  39. #include "xfs_dmapi.h"
  40. #include "xfs_mount.h"
  41. #include "xfs_trace.h"
  42. static kmem_zone_t *xfs_buf_zone;
  43. STATIC int xfsbufd(void *);
  44. STATIC int xfsbufd_wakeup(int, gfp_t);
  45. STATIC void xfs_buf_delwri_queue(xfs_buf_t *, int);
  46. static struct shrinker xfs_buf_shake = {
  47. .shrink = xfsbufd_wakeup,
  48. .seeks = DEFAULT_SEEKS,
  49. };
  50. static struct workqueue_struct *xfslogd_workqueue;
  51. struct workqueue_struct *xfsdatad_workqueue;
  52. struct workqueue_struct *xfsconvertd_workqueue;
  53. #ifdef XFS_BUF_LOCK_TRACKING
  54. # define XB_SET_OWNER(bp) ((bp)->b_last_holder = current->pid)
  55. # define XB_CLEAR_OWNER(bp) ((bp)->b_last_holder = -1)
  56. # define XB_GET_OWNER(bp) ((bp)->b_last_holder)
  57. #else
  58. # define XB_SET_OWNER(bp) do { } while (0)
  59. # define XB_CLEAR_OWNER(bp) do { } while (0)
  60. # define XB_GET_OWNER(bp) do { } while (0)
  61. #endif
  62. #define xb_to_gfp(flags) \
  63. ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : \
  64. ((flags) & XBF_DONT_BLOCK) ? GFP_NOFS : GFP_KERNEL) | __GFP_NOWARN)
  65. #define xb_to_km(flags) \
  66. (((flags) & XBF_DONT_BLOCK) ? KM_NOFS : KM_SLEEP)
  67. #define xfs_buf_allocate(flags) \
  68. kmem_zone_alloc(xfs_buf_zone, xb_to_km(flags))
  69. #define xfs_buf_deallocate(bp) \
  70. kmem_zone_free(xfs_buf_zone, (bp));
  71. /*
  72. * Page Region interfaces.
  73. *
  74. * For pages in filesystems where the blocksize is smaller than the
  75. * pagesize, we use the page->private field (long) to hold a bitmap
  76. * of uptodate regions within the page.
  77. *
  78. * Each such region is "bytes per page / bits per long" bytes long.
  79. *
  80. * NBPPR == number-of-bytes-per-page-region
  81. * BTOPR == bytes-to-page-region (rounded up)
  82. * BTOPRT == bytes-to-page-region-truncated (rounded down)
  83. */
  84. #if (BITS_PER_LONG == 32)
  85. #define PRSHIFT (PAGE_CACHE_SHIFT - 5) /* (32 == 1<<5) */
  86. #elif (BITS_PER_LONG == 64)
  87. #define PRSHIFT (PAGE_CACHE_SHIFT - 6) /* (64 == 1<<6) */
  88. #else
  89. #error BITS_PER_LONG must be 32 or 64
  90. #endif
  91. #define NBPPR (PAGE_CACHE_SIZE/BITS_PER_LONG)
  92. #define BTOPR(b) (((unsigned int)(b) + (NBPPR - 1)) >> PRSHIFT)
  93. #define BTOPRT(b) (((unsigned int)(b) >> PRSHIFT))
  94. STATIC unsigned long
  95. page_region_mask(
  96. size_t offset,
  97. size_t length)
  98. {
  99. unsigned long mask;
  100. int first, final;
  101. first = BTOPR(offset);
  102. final = BTOPRT(offset + length - 1);
  103. first = min(first, final);
  104. mask = ~0UL;
  105. mask <<= BITS_PER_LONG - (final - first);
  106. mask >>= BITS_PER_LONG - (final);
  107. ASSERT(offset + length <= PAGE_CACHE_SIZE);
  108. ASSERT((final - first) < BITS_PER_LONG && (final - first) >= 0);
  109. return mask;
  110. }
  111. STATIC void
  112. set_page_region(
  113. struct page *page,
  114. size_t offset,
  115. size_t length)
  116. {
  117. set_page_private(page,
  118. page_private(page) | page_region_mask(offset, length));
  119. if (page_private(page) == ~0UL)
  120. SetPageUptodate(page);
  121. }
  122. STATIC int
  123. test_page_region(
  124. struct page *page,
  125. size_t offset,
  126. size_t length)
  127. {
  128. unsigned long mask = page_region_mask(offset, length);
  129. return (mask && (page_private(page) & mask) == mask);
  130. }
  131. /*
  132. * Mapping of multi-page buffers into contiguous virtual space
  133. */
  134. typedef struct a_list {
  135. void *vm_addr;
  136. struct a_list *next;
  137. } a_list_t;
  138. static a_list_t *as_free_head;
  139. static int as_list_len;
  140. static DEFINE_SPINLOCK(as_lock);
  141. /*
  142. * Try to batch vunmaps because they are costly.
  143. */
  144. STATIC void
  145. free_address(
  146. void *addr)
  147. {
  148. a_list_t *aentry;
  149. #ifdef CONFIG_XEN
  150. /*
  151. * Xen needs to be able to make sure it can get an exclusive
  152. * RO mapping of pages it wants to turn into a pagetable. If
  153. * a newly allocated page is also still being vmap()ed by xfs,
  154. * it will cause pagetable construction to fail. This is a
  155. * quick workaround to always eagerly unmap pages so that Xen
  156. * is happy.
  157. */
  158. vunmap(addr);
  159. return;
  160. #endif
  161. aentry = kmalloc(sizeof(a_list_t), GFP_NOWAIT);
  162. if (likely(aentry)) {
  163. spin_lock(&as_lock);
  164. aentry->next = as_free_head;
  165. aentry->vm_addr = addr;
  166. as_free_head = aentry;
  167. as_list_len++;
  168. spin_unlock(&as_lock);
  169. } else {
  170. vunmap(addr);
  171. }
  172. }
  173. STATIC void
  174. purge_addresses(void)
  175. {
  176. a_list_t *aentry, *old;
  177. if (as_free_head == NULL)
  178. return;
  179. spin_lock(&as_lock);
  180. aentry = as_free_head;
  181. as_free_head = NULL;
  182. as_list_len = 0;
  183. spin_unlock(&as_lock);
  184. while ((old = aentry) != NULL) {
  185. vunmap(aentry->vm_addr);
  186. aentry = aentry->next;
  187. kfree(old);
  188. }
  189. }
  190. /*
  191. * Internal xfs_buf_t object manipulation
  192. */
  193. STATIC void
  194. _xfs_buf_initialize(
  195. xfs_buf_t *bp,
  196. xfs_buftarg_t *target,
  197. xfs_off_t range_base,
  198. size_t range_length,
  199. xfs_buf_flags_t flags)
  200. {
  201. /*
  202. * We don't want certain flags to appear in b_flags.
  203. */
  204. flags &= ~(XBF_LOCK|XBF_MAPPED|XBF_DONT_BLOCK|XBF_READ_AHEAD);
  205. memset(bp, 0, sizeof(xfs_buf_t));
  206. atomic_set(&bp->b_hold, 1);
  207. init_completion(&bp->b_iowait);
  208. INIT_LIST_HEAD(&bp->b_list);
  209. INIT_LIST_HEAD(&bp->b_hash_list);
  210. init_MUTEX_LOCKED(&bp->b_sema); /* held, no waiters */
  211. XB_SET_OWNER(bp);
  212. bp->b_target = target;
  213. bp->b_file_offset = range_base;
  214. /*
  215. * Set buffer_length and count_desired to the same value initially.
  216. * I/O routines should use count_desired, which will be the same in
  217. * most cases but may be reset (e.g. XFS recovery).
  218. */
  219. bp->b_buffer_length = bp->b_count_desired = range_length;
  220. bp->b_flags = flags;
  221. bp->b_bn = XFS_BUF_DADDR_NULL;
  222. atomic_set(&bp->b_pin_count, 0);
  223. init_waitqueue_head(&bp->b_waiters);
  224. XFS_STATS_INC(xb_create);
  225. trace_xfs_buf_init(bp, _RET_IP_);
  226. }
  227. /*
  228. * Allocate a page array capable of holding a specified number
  229. * of pages, and point the page buf at it.
  230. */
  231. STATIC int
  232. _xfs_buf_get_pages(
  233. xfs_buf_t *bp,
  234. int page_count,
  235. xfs_buf_flags_t flags)
  236. {
  237. /* Make sure that we have a page list */
  238. if (bp->b_pages == NULL) {
  239. bp->b_offset = xfs_buf_poff(bp->b_file_offset);
  240. bp->b_page_count = page_count;
  241. if (page_count <= XB_PAGES) {
  242. bp->b_pages = bp->b_page_array;
  243. } else {
  244. bp->b_pages = kmem_alloc(sizeof(struct page *) *
  245. page_count, xb_to_km(flags));
  246. if (bp->b_pages == NULL)
  247. return -ENOMEM;
  248. }
  249. memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
  250. }
  251. return 0;
  252. }
  253. /*
  254. * Frees b_pages if it was allocated.
  255. */
  256. STATIC void
  257. _xfs_buf_free_pages(
  258. xfs_buf_t *bp)
  259. {
  260. if (bp->b_pages != bp->b_page_array) {
  261. kmem_free(bp->b_pages);
  262. }
  263. }
  264. /*
  265. * Releases the specified buffer.
  266. *
  267. * The modification state of any associated pages is left unchanged.
  268. * The buffer most not be on any hash - use xfs_buf_rele instead for
  269. * hashed and refcounted buffers
  270. */
  271. void
  272. xfs_buf_free(
  273. xfs_buf_t *bp)
  274. {
  275. trace_xfs_buf_free(bp, _RET_IP_);
  276. ASSERT(list_empty(&bp->b_hash_list));
  277. if (bp->b_flags & (_XBF_PAGE_CACHE|_XBF_PAGES)) {
  278. uint i;
  279. if ((bp->b_flags & XBF_MAPPED) && (bp->b_page_count > 1))
  280. free_address(bp->b_addr - bp->b_offset);
  281. for (i = 0; i < bp->b_page_count; i++) {
  282. struct page *page = bp->b_pages[i];
  283. if (bp->b_flags & _XBF_PAGE_CACHE)
  284. ASSERT(!PagePrivate(page));
  285. page_cache_release(page);
  286. }
  287. _xfs_buf_free_pages(bp);
  288. }
  289. xfs_buf_deallocate(bp);
  290. }
  291. /*
  292. * Finds all pages for buffer in question and builds it's page list.
  293. */
  294. STATIC int
  295. _xfs_buf_lookup_pages(
  296. xfs_buf_t *bp,
  297. uint flags)
  298. {
  299. struct address_space *mapping = bp->b_target->bt_mapping;
  300. size_t blocksize = bp->b_target->bt_bsize;
  301. size_t size = bp->b_count_desired;
  302. size_t nbytes, offset;
  303. gfp_t gfp_mask = xb_to_gfp(flags);
  304. unsigned short page_count, i;
  305. pgoff_t first;
  306. xfs_off_t end;
  307. int error;
  308. end = bp->b_file_offset + bp->b_buffer_length;
  309. page_count = xfs_buf_btoc(end) - xfs_buf_btoct(bp->b_file_offset);
  310. error = _xfs_buf_get_pages(bp, page_count, flags);
  311. if (unlikely(error))
  312. return error;
  313. bp->b_flags |= _XBF_PAGE_CACHE;
  314. offset = bp->b_offset;
  315. first = bp->b_file_offset >> PAGE_CACHE_SHIFT;
  316. for (i = 0; i < bp->b_page_count; i++) {
  317. struct page *page;
  318. uint retries = 0;
  319. retry:
  320. page = find_or_create_page(mapping, first + i, gfp_mask);
  321. if (unlikely(page == NULL)) {
  322. if (flags & XBF_READ_AHEAD) {
  323. bp->b_page_count = i;
  324. for (i = 0; i < bp->b_page_count; i++)
  325. unlock_page(bp->b_pages[i]);
  326. return -ENOMEM;
  327. }
  328. /*
  329. * This could deadlock.
  330. *
  331. * But until all the XFS lowlevel code is revamped to
  332. * handle buffer allocation failures we can't do much.
  333. */
  334. if (!(++retries % 100))
  335. printk(KERN_ERR
  336. "XFS: possible memory allocation "
  337. "deadlock in %s (mode:0x%x)\n",
  338. __func__, gfp_mask);
  339. XFS_STATS_INC(xb_page_retries);
  340. xfsbufd_wakeup(0, gfp_mask);
  341. congestion_wait(BLK_RW_ASYNC, HZ/50);
  342. goto retry;
  343. }
  344. XFS_STATS_INC(xb_page_found);
  345. nbytes = min_t(size_t, size, PAGE_CACHE_SIZE - offset);
  346. size -= nbytes;
  347. ASSERT(!PagePrivate(page));
  348. if (!PageUptodate(page)) {
  349. page_count--;
  350. if (blocksize >= PAGE_CACHE_SIZE) {
  351. if (flags & XBF_READ)
  352. bp->b_flags |= _XBF_PAGE_LOCKED;
  353. } else if (!PagePrivate(page)) {
  354. if (test_page_region(page, offset, nbytes))
  355. page_count++;
  356. }
  357. }
  358. bp->b_pages[i] = page;
  359. offset = 0;
  360. }
  361. if (!(bp->b_flags & _XBF_PAGE_LOCKED)) {
  362. for (i = 0; i < bp->b_page_count; i++)
  363. unlock_page(bp->b_pages[i]);
  364. }
  365. if (page_count == bp->b_page_count)
  366. bp->b_flags |= XBF_DONE;
  367. return error;
  368. }
  369. /*
  370. * Map buffer into kernel address-space if nessecary.
  371. */
  372. STATIC int
  373. _xfs_buf_map_pages(
  374. xfs_buf_t *bp,
  375. uint flags)
  376. {
  377. /* A single page buffer is always mappable */
  378. if (bp->b_page_count == 1) {
  379. bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
  380. bp->b_flags |= XBF_MAPPED;
  381. } else if (flags & XBF_MAPPED) {
  382. if (as_list_len > 64)
  383. purge_addresses();
  384. bp->b_addr = vmap(bp->b_pages, bp->b_page_count,
  385. VM_MAP, PAGE_KERNEL);
  386. if (unlikely(bp->b_addr == NULL))
  387. return -ENOMEM;
  388. bp->b_addr += bp->b_offset;
  389. bp->b_flags |= XBF_MAPPED;
  390. }
  391. return 0;
  392. }
  393. /*
  394. * Finding and Reading Buffers
  395. */
  396. /*
  397. * Look up, and creates if absent, a lockable buffer for
  398. * a given range of an inode. The buffer is returned
  399. * locked. If other overlapping buffers exist, they are
  400. * released before the new buffer is created and locked,
  401. * which may imply that this call will block until those buffers
  402. * are unlocked. No I/O is implied by this call.
  403. */
  404. xfs_buf_t *
  405. _xfs_buf_find(
  406. xfs_buftarg_t *btp, /* block device target */
  407. xfs_off_t ioff, /* starting offset of range */
  408. size_t isize, /* length of range */
  409. xfs_buf_flags_t flags,
  410. xfs_buf_t *new_bp)
  411. {
  412. xfs_off_t range_base;
  413. size_t range_length;
  414. xfs_bufhash_t *hash;
  415. xfs_buf_t *bp, *n;
  416. range_base = (ioff << BBSHIFT);
  417. range_length = (isize << BBSHIFT);
  418. /* Check for IOs smaller than the sector size / not sector aligned */
  419. ASSERT(!(range_length < (1 << btp->bt_sshift)));
  420. ASSERT(!(range_base & (xfs_off_t)btp->bt_smask));
  421. hash = &btp->bt_hash[hash_long((unsigned long)ioff, btp->bt_hashshift)];
  422. spin_lock(&hash->bh_lock);
  423. list_for_each_entry_safe(bp, n, &hash->bh_list, b_hash_list) {
  424. ASSERT(btp == bp->b_target);
  425. if (bp->b_file_offset == range_base &&
  426. bp->b_buffer_length == range_length) {
  427. /*
  428. * If we look at something, bring it to the
  429. * front of the list for next time.
  430. */
  431. atomic_inc(&bp->b_hold);
  432. list_move(&bp->b_hash_list, &hash->bh_list);
  433. goto found;
  434. }
  435. }
  436. /* No match found */
  437. if (new_bp) {
  438. _xfs_buf_initialize(new_bp, btp, range_base,
  439. range_length, flags);
  440. new_bp->b_hash = hash;
  441. list_add(&new_bp->b_hash_list, &hash->bh_list);
  442. } else {
  443. XFS_STATS_INC(xb_miss_locked);
  444. }
  445. spin_unlock(&hash->bh_lock);
  446. return new_bp;
  447. found:
  448. spin_unlock(&hash->bh_lock);
  449. /* Attempt to get the semaphore without sleeping,
  450. * if this does not work then we need to drop the
  451. * spinlock and do a hard attempt on the semaphore.
  452. */
  453. if (down_trylock(&bp->b_sema)) {
  454. if (!(flags & XBF_TRYLOCK)) {
  455. /* wait for buffer ownership */
  456. xfs_buf_lock(bp);
  457. XFS_STATS_INC(xb_get_locked_waited);
  458. } else {
  459. /* We asked for a trylock and failed, no need
  460. * to look at file offset and length here, we
  461. * know that this buffer at least overlaps our
  462. * buffer and is locked, therefore our buffer
  463. * either does not exist, or is this buffer.
  464. */
  465. xfs_buf_rele(bp);
  466. XFS_STATS_INC(xb_busy_locked);
  467. return NULL;
  468. }
  469. } else {
  470. /* trylock worked */
  471. XB_SET_OWNER(bp);
  472. }
  473. if (bp->b_flags & XBF_STALE) {
  474. ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
  475. bp->b_flags &= XBF_MAPPED;
  476. }
  477. trace_xfs_buf_find(bp, flags, _RET_IP_);
  478. XFS_STATS_INC(xb_get_locked);
  479. return bp;
  480. }
  481. /*
  482. * Assembles a buffer covering the specified range.
  483. * Storage in memory for all portions of the buffer will be allocated,
  484. * although backing storage may not be.
  485. */
  486. xfs_buf_t *
  487. xfs_buf_get(
  488. xfs_buftarg_t *target,/* target for buffer */
  489. xfs_off_t ioff, /* starting offset of range */
  490. size_t isize, /* length of range */
  491. xfs_buf_flags_t flags)
  492. {
  493. xfs_buf_t *bp, *new_bp;
  494. int error = 0, i;
  495. new_bp = xfs_buf_allocate(flags);
  496. if (unlikely(!new_bp))
  497. return NULL;
  498. bp = _xfs_buf_find(target, ioff, isize, flags, new_bp);
  499. if (bp == new_bp) {
  500. error = _xfs_buf_lookup_pages(bp, flags);
  501. if (error)
  502. goto no_buffer;
  503. } else {
  504. xfs_buf_deallocate(new_bp);
  505. if (unlikely(bp == NULL))
  506. return NULL;
  507. }
  508. for (i = 0; i < bp->b_page_count; i++)
  509. mark_page_accessed(bp->b_pages[i]);
  510. if (!(bp->b_flags & XBF_MAPPED)) {
  511. error = _xfs_buf_map_pages(bp, flags);
  512. if (unlikely(error)) {
  513. printk(KERN_WARNING "%s: failed to map pages\n",
  514. __func__);
  515. goto no_buffer;
  516. }
  517. }
  518. XFS_STATS_INC(xb_get);
  519. /*
  520. * Always fill in the block number now, the mapped cases can do
  521. * their own overlay of this later.
  522. */
  523. bp->b_bn = ioff;
  524. bp->b_count_desired = bp->b_buffer_length;
  525. trace_xfs_buf_get(bp, flags, _RET_IP_);
  526. return bp;
  527. no_buffer:
  528. if (flags & (XBF_LOCK | XBF_TRYLOCK))
  529. xfs_buf_unlock(bp);
  530. xfs_buf_rele(bp);
  531. return NULL;
  532. }
  533. STATIC int
  534. _xfs_buf_read(
  535. xfs_buf_t *bp,
  536. xfs_buf_flags_t flags)
  537. {
  538. int status;
  539. ASSERT(!(flags & (XBF_DELWRI|XBF_WRITE)));
  540. ASSERT(bp->b_bn != XFS_BUF_DADDR_NULL);
  541. bp->b_flags &= ~(XBF_WRITE | XBF_ASYNC | XBF_DELWRI | \
  542. XBF_READ_AHEAD | _XBF_RUN_QUEUES);
  543. bp->b_flags |= flags & (XBF_READ | XBF_ASYNC | \
  544. XBF_READ_AHEAD | _XBF_RUN_QUEUES);
  545. status = xfs_buf_iorequest(bp);
  546. if (!status && !(flags & XBF_ASYNC))
  547. status = xfs_buf_iowait(bp);
  548. return status;
  549. }
  550. xfs_buf_t *
  551. xfs_buf_read(
  552. xfs_buftarg_t *target,
  553. xfs_off_t ioff,
  554. size_t isize,
  555. xfs_buf_flags_t flags)
  556. {
  557. xfs_buf_t *bp;
  558. flags |= XBF_READ;
  559. bp = xfs_buf_get(target, ioff, isize, flags);
  560. if (bp) {
  561. trace_xfs_buf_read(bp, flags, _RET_IP_);
  562. if (!XFS_BUF_ISDONE(bp)) {
  563. XFS_STATS_INC(xb_get_read);
  564. _xfs_buf_read(bp, flags);
  565. } else if (flags & XBF_ASYNC) {
  566. /*
  567. * Read ahead call which is already satisfied,
  568. * drop the buffer
  569. */
  570. goto no_buffer;
  571. } else {
  572. /* We do not want read in the flags */
  573. bp->b_flags &= ~XBF_READ;
  574. }
  575. }
  576. return bp;
  577. no_buffer:
  578. if (flags & (XBF_LOCK | XBF_TRYLOCK))
  579. xfs_buf_unlock(bp);
  580. xfs_buf_rele(bp);
  581. return NULL;
  582. }
  583. /*
  584. * If we are not low on memory then do the readahead in a deadlock
  585. * safe manner.
  586. */
  587. void
  588. xfs_buf_readahead(
  589. xfs_buftarg_t *target,
  590. xfs_off_t ioff,
  591. size_t isize,
  592. xfs_buf_flags_t flags)
  593. {
  594. struct backing_dev_info *bdi;
  595. bdi = target->bt_mapping->backing_dev_info;
  596. if (bdi_read_congested(bdi))
  597. return;
  598. flags |= (XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD);
  599. xfs_buf_read(target, ioff, isize, flags);
  600. }
  601. xfs_buf_t *
  602. xfs_buf_get_empty(
  603. size_t len,
  604. xfs_buftarg_t *target)
  605. {
  606. xfs_buf_t *bp;
  607. bp = xfs_buf_allocate(0);
  608. if (bp)
  609. _xfs_buf_initialize(bp, target, 0, len, 0);
  610. return bp;
  611. }
  612. static inline struct page *
  613. mem_to_page(
  614. void *addr)
  615. {
  616. if ((!is_vmalloc_addr(addr))) {
  617. return virt_to_page(addr);
  618. } else {
  619. return vmalloc_to_page(addr);
  620. }
  621. }
  622. int
  623. xfs_buf_associate_memory(
  624. xfs_buf_t *bp,
  625. void *mem,
  626. size_t len)
  627. {
  628. int rval;
  629. int i = 0;
  630. unsigned long pageaddr;
  631. unsigned long offset;
  632. size_t buflen;
  633. int page_count;
  634. pageaddr = (unsigned long)mem & PAGE_CACHE_MASK;
  635. offset = (unsigned long)mem - pageaddr;
  636. buflen = PAGE_CACHE_ALIGN(len + offset);
  637. page_count = buflen >> PAGE_CACHE_SHIFT;
  638. /* Free any previous set of page pointers */
  639. if (bp->b_pages)
  640. _xfs_buf_free_pages(bp);
  641. bp->b_pages = NULL;
  642. bp->b_addr = mem;
  643. rval = _xfs_buf_get_pages(bp, page_count, XBF_DONT_BLOCK);
  644. if (rval)
  645. return rval;
  646. bp->b_offset = offset;
  647. for (i = 0; i < bp->b_page_count; i++) {
  648. bp->b_pages[i] = mem_to_page((void *)pageaddr);
  649. pageaddr += PAGE_CACHE_SIZE;
  650. }
  651. bp->b_count_desired = len;
  652. bp->b_buffer_length = buflen;
  653. bp->b_flags |= XBF_MAPPED;
  654. bp->b_flags &= ~_XBF_PAGE_LOCKED;
  655. return 0;
  656. }
  657. xfs_buf_t *
  658. xfs_buf_get_noaddr(
  659. size_t len,
  660. xfs_buftarg_t *target)
  661. {
  662. unsigned long page_count = PAGE_ALIGN(len) >> PAGE_SHIFT;
  663. int error, i;
  664. xfs_buf_t *bp;
  665. bp = xfs_buf_allocate(0);
  666. if (unlikely(bp == NULL))
  667. goto fail;
  668. _xfs_buf_initialize(bp, target, 0, len, 0);
  669. error = _xfs_buf_get_pages(bp, page_count, 0);
  670. if (error)
  671. goto fail_free_buf;
  672. for (i = 0; i < page_count; i++) {
  673. bp->b_pages[i] = alloc_page(GFP_KERNEL);
  674. if (!bp->b_pages[i])
  675. goto fail_free_mem;
  676. }
  677. bp->b_flags |= _XBF_PAGES;
  678. error = _xfs_buf_map_pages(bp, XBF_MAPPED);
  679. if (unlikely(error)) {
  680. printk(KERN_WARNING "%s: failed to map pages\n",
  681. __func__);
  682. goto fail_free_mem;
  683. }
  684. xfs_buf_unlock(bp);
  685. trace_xfs_buf_get_noaddr(bp, _RET_IP_);
  686. return bp;
  687. fail_free_mem:
  688. while (--i >= 0)
  689. __free_page(bp->b_pages[i]);
  690. _xfs_buf_free_pages(bp);
  691. fail_free_buf:
  692. xfs_buf_deallocate(bp);
  693. fail:
  694. return NULL;
  695. }
  696. /*
  697. * Increment reference count on buffer, to hold the buffer concurrently
  698. * with another thread which may release (free) the buffer asynchronously.
  699. * Must hold the buffer already to call this function.
  700. */
  701. void
  702. xfs_buf_hold(
  703. xfs_buf_t *bp)
  704. {
  705. trace_xfs_buf_hold(bp, _RET_IP_);
  706. atomic_inc(&bp->b_hold);
  707. }
  708. /*
  709. * Releases a hold on the specified buffer. If the
  710. * the hold count is 1, calls xfs_buf_free.
  711. */
  712. void
  713. xfs_buf_rele(
  714. xfs_buf_t *bp)
  715. {
  716. xfs_bufhash_t *hash = bp->b_hash;
  717. trace_xfs_buf_rele(bp, _RET_IP_);
  718. if (unlikely(!hash)) {
  719. ASSERT(!bp->b_relse);
  720. if (atomic_dec_and_test(&bp->b_hold))
  721. xfs_buf_free(bp);
  722. return;
  723. }
  724. ASSERT(atomic_read(&bp->b_hold) > 0);
  725. if (atomic_dec_and_lock(&bp->b_hold, &hash->bh_lock)) {
  726. if (bp->b_relse) {
  727. atomic_inc(&bp->b_hold);
  728. spin_unlock(&hash->bh_lock);
  729. (*(bp->b_relse)) (bp);
  730. } else if (bp->b_flags & XBF_FS_MANAGED) {
  731. spin_unlock(&hash->bh_lock);
  732. } else {
  733. ASSERT(!(bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)));
  734. list_del_init(&bp->b_hash_list);
  735. spin_unlock(&hash->bh_lock);
  736. xfs_buf_free(bp);
  737. }
  738. }
  739. }
  740. /*
  741. * Mutual exclusion on buffers. Locking model:
  742. *
  743. * Buffers associated with inodes for which buffer locking
  744. * is not enabled are not protected by semaphores, and are
  745. * assumed to be exclusively owned by the caller. There is a
  746. * spinlock in the buffer, used by the caller when concurrent
  747. * access is possible.
  748. */
  749. /*
  750. * Locks a buffer object, if it is not already locked.
  751. * Note that this in no way locks the underlying pages, so it is only
  752. * useful for synchronizing concurrent use of buffer objects, not for
  753. * synchronizing independent access to the underlying pages.
  754. */
  755. int
  756. xfs_buf_cond_lock(
  757. xfs_buf_t *bp)
  758. {
  759. int locked;
  760. locked = down_trylock(&bp->b_sema) == 0;
  761. if (locked)
  762. XB_SET_OWNER(bp);
  763. trace_xfs_buf_cond_lock(bp, _RET_IP_);
  764. return locked ? 0 : -EBUSY;
  765. }
  766. int
  767. xfs_buf_lock_value(
  768. xfs_buf_t *bp)
  769. {
  770. return bp->b_sema.count;
  771. }
  772. /*
  773. * Locks a buffer object.
  774. * Note that this in no way locks the underlying pages, so it is only
  775. * useful for synchronizing concurrent use of buffer objects, not for
  776. * synchronizing independent access to the underlying pages.
  777. */
  778. void
  779. xfs_buf_lock(
  780. xfs_buf_t *bp)
  781. {
  782. trace_xfs_buf_lock(bp, _RET_IP_);
  783. if (atomic_read(&bp->b_io_remaining))
  784. blk_run_address_space(bp->b_target->bt_mapping);
  785. down(&bp->b_sema);
  786. XB_SET_OWNER(bp);
  787. trace_xfs_buf_lock_done(bp, _RET_IP_);
  788. }
  789. /*
  790. * Releases the lock on the buffer object.
  791. * If the buffer is marked delwri but is not queued, do so before we
  792. * unlock the buffer as we need to set flags correctly. We also need to
  793. * take a reference for the delwri queue because the unlocker is going to
  794. * drop their's and they don't know we just queued it.
  795. */
  796. void
  797. xfs_buf_unlock(
  798. xfs_buf_t *bp)
  799. {
  800. if ((bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)) == XBF_DELWRI) {
  801. atomic_inc(&bp->b_hold);
  802. bp->b_flags |= XBF_ASYNC;
  803. xfs_buf_delwri_queue(bp, 0);
  804. }
  805. XB_CLEAR_OWNER(bp);
  806. up(&bp->b_sema);
  807. trace_xfs_buf_unlock(bp, _RET_IP_);
  808. }
  809. /*
  810. * Pinning Buffer Storage in Memory
  811. * Ensure that no attempt to force a buffer to disk will succeed.
  812. */
  813. void
  814. xfs_buf_pin(
  815. xfs_buf_t *bp)
  816. {
  817. trace_xfs_buf_pin(bp, _RET_IP_);
  818. atomic_inc(&bp->b_pin_count);
  819. }
  820. void
  821. xfs_buf_unpin(
  822. xfs_buf_t *bp)
  823. {
  824. trace_xfs_buf_unpin(bp, _RET_IP_);
  825. if (atomic_dec_and_test(&bp->b_pin_count))
  826. wake_up_all(&bp->b_waiters);
  827. }
  828. int
  829. xfs_buf_ispin(
  830. xfs_buf_t *bp)
  831. {
  832. return atomic_read(&bp->b_pin_count);
  833. }
  834. STATIC void
  835. xfs_buf_wait_unpin(
  836. xfs_buf_t *bp)
  837. {
  838. DECLARE_WAITQUEUE (wait, current);
  839. if (atomic_read(&bp->b_pin_count) == 0)
  840. return;
  841. add_wait_queue(&bp->b_waiters, &wait);
  842. for (;;) {
  843. set_current_state(TASK_UNINTERRUPTIBLE);
  844. if (atomic_read(&bp->b_pin_count) == 0)
  845. break;
  846. if (atomic_read(&bp->b_io_remaining))
  847. blk_run_address_space(bp->b_target->bt_mapping);
  848. schedule();
  849. }
  850. remove_wait_queue(&bp->b_waiters, &wait);
  851. set_current_state(TASK_RUNNING);
  852. }
  853. /*
  854. * Buffer Utility Routines
  855. */
  856. STATIC void
  857. xfs_buf_iodone_work(
  858. struct work_struct *work)
  859. {
  860. xfs_buf_t *bp =
  861. container_of(work, xfs_buf_t, b_iodone_work);
  862. /*
  863. * We can get an EOPNOTSUPP to ordered writes. Here we clear the
  864. * ordered flag and reissue them. Because we can't tell the higher
  865. * layers directly that they should not issue ordered I/O anymore, they
  866. * need to check if the _XFS_BARRIER_FAILED flag was set during I/O completion.
  867. */
  868. if ((bp->b_error == EOPNOTSUPP) &&
  869. (bp->b_flags & (XBF_ORDERED|XBF_ASYNC)) == (XBF_ORDERED|XBF_ASYNC)) {
  870. trace_xfs_buf_ordered_retry(bp, _RET_IP_);
  871. bp->b_flags &= ~XBF_ORDERED;
  872. bp->b_flags |= _XFS_BARRIER_FAILED;
  873. xfs_buf_iorequest(bp);
  874. } else if (bp->b_iodone)
  875. (*(bp->b_iodone))(bp);
  876. else if (bp->b_flags & XBF_ASYNC)
  877. xfs_buf_relse(bp);
  878. }
  879. void
  880. xfs_buf_ioend(
  881. xfs_buf_t *bp,
  882. int schedule)
  883. {
  884. trace_xfs_buf_iodone(bp, _RET_IP_);
  885. bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
  886. if (bp->b_error == 0)
  887. bp->b_flags |= XBF_DONE;
  888. if ((bp->b_iodone) || (bp->b_flags & XBF_ASYNC)) {
  889. if (schedule) {
  890. INIT_WORK(&bp->b_iodone_work, xfs_buf_iodone_work);
  891. queue_work(xfslogd_workqueue, &bp->b_iodone_work);
  892. } else {
  893. xfs_buf_iodone_work(&bp->b_iodone_work);
  894. }
  895. } else {
  896. complete(&bp->b_iowait);
  897. }
  898. }
  899. void
  900. xfs_buf_ioerror(
  901. xfs_buf_t *bp,
  902. int error)
  903. {
  904. ASSERT(error >= 0 && error <= 0xffff);
  905. bp->b_error = (unsigned short)error;
  906. trace_xfs_buf_ioerror(bp, error, _RET_IP_);
  907. }
  908. int
  909. xfs_bawrite(
  910. void *mp,
  911. struct xfs_buf *bp)
  912. {
  913. trace_xfs_buf_bawrite(bp, _RET_IP_);
  914. ASSERT(bp->b_bn != XFS_BUF_DADDR_NULL);
  915. xfs_buf_delwri_dequeue(bp);
  916. bp->b_flags &= ~(XBF_READ | XBF_DELWRI | XBF_READ_AHEAD);
  917. bp->b_flags |= (XBF_WRITE | XBF_ASYNC | _XBF_RUN_QUEUES);
  918. bp->b_mount = mp;
  919. bp->b_strat = xfs_bdstrat_cb;
  920. return xfs_bdstrat_cb(bp);
  921. }
  922. void
  923. xfs_bdwrite(
  924. void *mp,
  925. struct xfs_buf *bp)
  926. {
  927. trace_xfs_buf_bdwrite(bp, _RET_IP_);
  928. bp->b_strat = xfs_bdstrat_cb;
  929. bp->b_mount = mp;
  930. bp->b_flags &= ~XBF_READ;
  931. bp->b_flags |= (XBF_DELWRI | XBF_ASYNC);
  932. xfs_buf_delwri_queue(bp, 1);
  933. }
  934. STATIC void
  935. _xfs_buf_ioend(
  936. xfs_buf_t *bp,
  937. int schedule)
  938. {
  939. if (atomic_dec_and_test(&bp->b_io_remaining) == 1) {
  940. bp->b_flags &= ~_XBF_PAGE_LOCKED;
  941. xfs_buf_ioend(bp, schedule);
  942. }
  943. }
  944. STATIC void
  945. xfs_buf_bio_end_io(
  946. struct bio *bio,
  947. int error)
  948. {
  949. xfs_buf_t *bp = (xfs_buf_t *)bio->bi_private;
  950. unsigned int blocksize = bp->b_target->bt_bsize;
  951. struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
  952. xfs_buf_ioerror(bp, -error);
  953. do {
  954. struct page *page = bvec->bv_page;
  955. ASSERT(!PagePrivate(page));
  956. if (unlikely(bp->b_error)) {
  957. if (bp->b_flags & XBF_READ)
  958. ClearPageUptodate(page);
  959. } else if (blocksize >= PAGE_CACHE_SIZE) {
  960. SetPageUptodate(page);
  961. } else if (!PagePrivate(page) &&
  962. (bp->b_flags & _XBF_PAGE_CACHE)) {
  963. set_page_region(page, bvec->bv_offset, bvec->bv_len);
  964. }
  965. if (--bvec >= bio->bi_io_vec)
  966. prefetchw(&bvec->bv_page->flags);
  967. if (bp->b_flags & _XBF_PAGE_LOCKED)
  968. unlock_page(page);
  969. } while (bvec >= bio->bi_io_vec);
  970. _xfs_buf_ioend(bp, 1);
  971. bio_put(bio);
  972. }
  973. STATIC void
  974. _xfs_buf_ioapply(
  975. xfs_buf_t *bp)
  976. {
  977. int rw, map_i, total_nr_pages, nr_pages;
  978. struct bio *bio;
  979. int offset = bp->b_offset;
  980. int size = bp->b_count_desired;
  981. sector_t sector = bp->b_bn;
  982. unsigned int blocksize = bp->b_target->bt_bsize;
  983. total_nr_pages = bp->b_page_count;
  984. map_i = 0;
  985. if (bp->b_flags & XBF_ORDERED) {
  986. ASSERT(!(bp->b_flags & XBF_READ));
  987. rw = WRITE_BARRIER;
  988. } else if (bp->b_flags & _XBF_RUN_QUEUES) {
  989. ASSERT(!(bp->b_flags & XBF_READ_AHEAD));
  990. bp->b_flags &= ~_XBF_RUN_QUEUES;
  991. rw = (bp->b_flags & XBF_WRITE) ? WRITE_SYNC : READ_SYNC;
  992. } else {
  993. rw = (bp->b_flags & XBF_WRITE) ? WRITE :
  994. (bp->b_flags & XBF_READ_AHEAD) ? READA : READ;
  995. }
  996. /* Special code path for reading a sub page size buffer in --
  997. * we populate up the whole page, and hence the other metadata
  998. * in the same page. This optimization is only valid when the
  999. * filesystem block size is not smaller than the page size.
  1000. */
  1001. if ((bp->b_buffer_length < PAGE_CACHE_SIZE) &&
  1002. ((bp->b_flags & (XBF_READ|_XBF_PAGE_LOCKED)) ==
  1003. (XBF_READ|_XBF_PAGE_LOCKED)) &&
  1004. (blocksize >= PAGE_CACHE_SIZE)) {
  1005. bio = bio_alloc(GFP_NOIO, 1);
  1006. bio->bi_bdev = bp->b_target->bt_bdev;
  1007. bio->bi_sector = sector - (offset >> BBSHIFT);
  1008. bio->bi_end_io = xfs_buf_bio_end_io;
  1009. bio->bi_private = bp;
  1010. bio_add_page(bio, bp->b_pages[0], PAGE_CACHE_SIZE, 0);
  1011. size = 0;
  1012. atomic_inc(&bp->b_io_remaining);
  1013. goto submit_io;
  1014. }
  1015. next_chunk:
  1016. atomic_inc(&bp->b_io_remaining);
  1017. nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
  1018. if (nr_pages > total_nr_pages)
  1019. nr_pages = total_nr_pages;
  1020. bio = bio_alloc(GFP_NOIO, nr_pages);
  1021. bio->bi_bdev = bp->b_target->bt_bdev;
  1022. bio->bi_sector = sector;
  1023. bio->bi_end_io = xfs_buf_bio_end_io;
  1024. bio->bi_private = bp;
  1025. for (; size && nr_pages; nr_pages--, map_i++) {
  1026. int rbytes, nbytes = PAGE_CACHE_SIZE - offset;
  1027. if (nbytes > size)
  1028. nbytes = size;
  1029. rbytes = bio_add_page(bio, bp->b_pages[map_i], nbytes, offset);
  1030. if (rbytes < nbytes)
  1031. break;
  1032. offset = 0;
  1033. sector += nbytes >> BBSHIFT;
  1034. size -= nbytes;
  1035. total_nr_pages--;
  1036. }
  1037. submit_io:
  1038. if (likely(bio->bi_size)) {
  1039. submit_bio(rw, bio);
  1040. if (size)
  1041. goto next_chunk;
  1042. } else {
  1043. bio_put(bio);
  1044. xfs_buf_ioerror(bp, EIO);
  1045. }
  1046. }
  1047. int
  1048. xfs_buf_iorequest(
  1049. xfs_buf_t *bp)
  1050. {
  1051. trace_xfs_buf_iorequest(bp, _RET_IP_);
  1052. if (bp->b_flags & XBF_DELWRI) {
  1053. xfs_buf_delwri_queue(bp, 1);
  1054. return 0;
  1055. }
  1056. if (bp->b_flags & XBF_WRITE) {
  1057. xfs_buf_wait_unpin(bp);
  1058. }
  1059. xfs_buf_hold(bp);
  1060. /* Set the count to 1 initially, this will stop an I/O
  1061. * completion callout which happens before we have started
  1062. * all the I/O from calling xfs_buf_ioend too early.
  1063. */
  1064. atomic_set(&bp->b_io_remaining, 1);
  1065. _xfs_buf_ioapply(bp);
  1066. _xfs_buf_ioend(bp, 0);
  1067. xfs_buf_rele(bp);
  1068. return 0;
  1069. }
  1070. /*
  1071. * Waits for I/O to complete on the buffer supplied.
  1072. * It returns immediately if no I/O is pending.
  1073. * It returns the I/O error code, if any, or 0 if there was no error.
  1074. */
  1075. int
  1076. xfs_buf_iowait(
  1077. xfs_buf_t *bp)
  1078. {
  1079. trace_xfs_buf_iowait(bp, _RET_IP_);
  1080. if (atomic_read(&bp->b_io_remaining))
  1081. blk_run_address_space(bp->b_target->bt_mapping);
  1082. wait_for_completion(&bp->b_iowait);
  1083. trace_xfs_buf_iowait_done(bp, _RET_IP_);
  1084. return bp->b_error;
  1085. }
  1086. xfs_caddr_t
  1087. xfs_buf_offset(
  1088. xfs_buf_t *bp,
  1089. size_t offset)
  1090. {
  1091. struct page *page;
  1092. if (bp->b_flags & XBF_MAPPED)
  1093. return XFS_BUF_PTR(bp) + offset;
  1094. offset += bp->b_offset;
  1095. page = bp->b_pages[offset >> PAGE_CACHE_SHIFT];
  1096. return (xfs_caddr_t)page_address(page) + (offset & (PAGE_CACHE_SIZE-1));
  1097. }
  1098. /*
  1099. * Move data into or out of a buffer.
  1100. */
  1101. void
  1102. xfs_buf_iomove(
  1103. xfs_buf_t *bp, /* buffer to process */
  1104. size_t boff, /* starting buffer offset */
  1105. size_t bsize, /* length to copy */
  1106. caddr_t data, /* data address */
  1107. xfs_buf_rw_t mode) /* read/write/zero flag */
  1108. {
  1109. size_t bend, cpoff, csize;
  1110. struct page *page;
  1111. bend = boff + bsize;
  1112. while (boff < bend) {
  1113. page = bp->b_pages[xfs_buf_btoct(boff + bp->b_offset)];
  1114. cpoff = xfs_buf_poff(boff + bp->b_offset);
  1115. csize = min_t(size_t,
  1116. PAGE_CACHE_SIZE-cpoff, bp->b_count_desired-boff);
  1117. ASSERT(((csize + cpoff) <= PAGE_CACHE_SIZE));
  1118. switch (mode) {
  1119. case XBRW_ZERO:
  1120. memset(page_address(page) + cpoff, 0, csize);
  1121. break;
  1122. case XBRW_READ:
  1123. memcpy(data, page_address(page) + cpoff, csize);
  1124. break;
  1125. case XBRW_WRITE:
  1126. memcpy(page_address(page) + cpoff, data, csize);
  1127. }
  1128. boff += csize;
  1129. data += csize;
  1130. }
  1131. }
  1132. /*
  1133. * Handling of buffer targets (buftargs).
  1134. */
  1135. /*
  1136. * Wait for any bufs with callbacks that have been submitted but
  1137. * have not yet returned... walk the hash list for the target.
  1138. */
  1139. void
  1140. xfs_wait_buftarg(
  1141. xfs_buftarg_t *btp)
  1142. {
  1143. xfs_buf_t *bp, *n;
  1144. xfs_bufhash_t *hash;
  1145. uint i;
  1146. for (i = 0; i < (1 << btp->bt_hashshift); i++) {
  1147. hash = &btp->bt_hash[i];
  1148. again:
  1149. spin_lock(&hash->bh_lock);
  1150. list_for_each_entry_safe(bp, n, &hash->bh_list, b_hash_list) {
  1151. ASSERT(btp == bp->b_target);
  1152. if (!(bp->b_flags & XBF_FS_MANAGED)) {
  1153. spin_unlock(&hash->bh_lock);
  1154. /*
  1155. * Catch superblock reference count leaks
  1156. * immediately
  1157. */
  1158. BUG_ON(bp->b_bn == 0);
  1159. delay(100);
  1160. goto again;
  1161. }
  1162. }
  1163. spin_unlock(&hash->bh_lock);
  1164. }
  1165. }
  1166. /*
  1167. * Allocate buffer hash table for a given target.
  1168. * For devices containing metadata (i.e. not the log/realtime devices)
  1169. * we need to allocate a much larger hash table.
  1170. */
  1171. STATIC void
  1172. xfs_alloc_bufhash(
  1173. xfs_buftarg_t *btp,
  1174. int external)
  1175. {
  1176. unsigned int i;
  1177. btp->bt_hashshift = external ? 3 : 8; /* 8 or 256 buckets */
  1178. btp->bt_hashmask = (1 << btp->bt_hashshift) - 1;
  1179. btp->bt_hash = kmem_zalloc((1 << btp->bt_hashshift) *
  1180. sizeof(xfs_bufhash_t), KM_SLEEP | KM_LARGE);
  1181. for (i = 0; i < (1 << btp->bt_hashshift); i++) {
  1182. spin_lock_init(&btp->bt_hash[i].bh_lock);
  1183. INIT_LIST_HEAD(&btp->bt_hash[i].bh_list);
  1184. }
  1185. }
  1186. STATIC void
  1187. xfs_free_bufhash(
  1188. xfs_buftarg_t *btp)
  1189. {
  1190. kmem_free(btp->bt_hash);
  1191. btp->bt_hash = NULL;
  1192. }
  1193. /*
  1194. * buftarg list for delwrite queue processing
  1195. */
  1196. static LIST_HEAD(xfs_buftarg_list);
  1197. static DEFINE_SPINLOCK(xfs_buftarg_lock);
  1198. STATIC void
  1199. xfs_register_buftarg(
  1200. xfs_buftarg_t *btp)
  1201. {
  1202. spin_lock(&xfs_buftarg_lock);
  1203. list_add(&btp->bt_list, &xfs_buftarg_list);
  1204. spin_unlock(&xfs_buftarg_lock);
  1205. }
  1206. STATIC void
  1207. xfs_unregister_buftarg(
  1208. xfs_buftarg_t *btp)
  1209. {
  1210. spin_lock(&xfs_buftarg_lock);
  1211. list_del(&btp->bt_list);
  1212. spin_unlock(&xfs_buftarg_lock);
  1213. }
  1214. void
  1215. xfs_free_buftarg(
  1216. struct xfs_mount *mp,
  1217. struct xfs_buftarg *btp)
  1218. {
  1219. xfs_flush_buftarg(btp, 1);
  1220. if (mp->m_flags & XFS_MOUNT_BARRIER)
  1221. xfs_blkdev_issue_flush(btp);
  1222. xfs_free_bufhash(btp);
  1223. iput(btp->bt_mapping->host);
  1224. /* Unregister the buftarg first so that we don't get a
  1225. * wakeup finding a non-existent task
  1226. */
  1227. xfs_unregister_buftarg(btp);
  1228. kthread_stop(btp->bt_task);
  1229. kmem_free(btp);
  1230. }
  1231. STATIC int
  1232. xfs_setsize_buftarg_flags(
  1233. xfs_buftarg_t *btp,
  1234. unsigned int blocksize,
  1235. unsigned int sectorsize,
  1236. int verbose)
  1237. {
  1238. btp->bt_bsize = blocksize;
  1239. btp->bt_sshift = ffs(sectorsize) - 1;
  1240. btp->bt_smask = sectorsize - 1;
  1241. if (set_blocksize(btp->bt_bdev, sectorsize)) {
  1242. printk(KERN_WARNING
  1243. "XFS: Cannot set_blocksize to %u on device %s\n",
  1244. sectorsize, XFS_BUFTARG_NAME(btp));
  1245. return EINVAL;
  1246. }
  1247. if (verbose &&
  1248. (PAGE_CACHE_SIZE / BITS_PER_LONG) > sectorsize) {
  1249. printk(KERN_WARNING
  1250. "XFS: %u byte sectors in use on device %s. "
  1251. "This is suboptimal; %u or greater is ideal.\n",
  1252. sectorsize, XFS_BUFTARG_NAME(btp),
  1253. (unsigned int)PAGE_CACHE_SIZE / BITS_PER_LONG);
  1254. }
  1255. return 0;
  1256. }
  1257. /*
  1258. * When allocating the initial buffer target we have not yet
  1259. * read in the superblock, so don't know what sized sectors
  1260. * are being used is at this early stage. Play safe.
  1261. */
  1262. STATIC int
  1263. xfs_setsize_buftarg_early(
  1264. xfs_buftarg_t *btp,
  1265. struct block_device *bdev)
  1266. {
  1267. return xfs_setsize_buftarg_flags(btp,
  1268. PAGE_CACHE_SIZE, bdev_logical_block_size(bdev), 0);
  1269. }
  1270. int
  1271. xfs_setsize_buftarg(
  1272. xfs_buftarg_t *btp,
  1273. unsigned int blocksize,
  1274. unsigned int sectorsize)
  1275. {
  1276. return xfs_setsize_buftarg_flags(btp, blocksize, sectorsize, 1);
  1277. }
  1278. STATIC int
  1279. xfs_mapping_buftarg(
  1280. xfs_buftarg_t *btp,
  1281. struct block_device *bdev)
  1282. {
  1283. struct backing_dev_info *bdi;
  1284. struct inode *inode;
  1285. struct address_space *mapping;
  1286. static const struct address_space_operations mapping_aops = {
  1287. .sync_page = block_sync_page,
  1288. .migratepage = fail_migrate_page,
  1289. };
  1290. inode = new_inode(bdev->bd_inode->i_sb);
  1291. if (!inode) {
  1292. printk(KERN_WARNING
  1293. "XFS: Cannot allocate mapping inode for device %s\n",
  1294. XFS_BUFTARG_NAME(btp));
  1295. return ENOMEM;
  1296. }
  1297. inode->i_mode = S_IFBLK;
  1298. inode->i_bdev = bdev;
  1299. inode->i_rdev = bdev->bd_dev;
  1300. bdi = blk_get_backing_dev_info(bdev);
  1301. if (!bdi)
  1302. bdi = &default_backing_dev_info;
  1303. mapping = &inode->i_data;
  1304. mapping->a_ops = &mapping_aops;
  1305. mapping->backing_dev_info = bdi;
  1306. mapping_set_gfp_mask(mapping, GFP_NOFS);
  1307. btp->bt_mapping = mapping;
  1308. return 0;
  1309. }
  1310. STATIC int
  1311. xfs_alloc_delwrite_queue(
  1312. xfs_buftarg_t *btp)
  1313. {
  1314. int error = 0;
  1315. INIT_LIST_HEAD(&btp->bt_list);
  1316. INIT_LIST_HEAD(&btp->bt_delwrite_queue);
  1317. spin_lock_init(&btp->bt_delwrite_lock);
  1318. btp->bt_flags = 0;
  1319. btp->bt_task = kthread_run(xfsbufd, btp, "xfsbufd");
  1320. if (IS_ERR(btp->bt_task)) {
  1321. error = PTR_ERR(btp->bt_task);
  1322. goto out_error;
  1323. }
  1324. xfs_register_buftarg(btp);
  1325. out_error:
  1326. return error;
  1327. }
  1328. xfs_buftarg_t *
  1329. xfs_alloc_buftarg(
  1330. struct block_device *bdev,
  1331. int external)
  1332. {
  1333. xfs_buftarg_t *btp;
  1334. btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
  1335. btp->bt_dev = bdev->bd_dev;
  1336. btp->bt_bdev = bdev;
  1337. if (xfs_setsize_buftarg_early(btp, bdev))
  1338. goto error;
  1339. if (xfs_mapping_buftarg(btp, bdev))
  1340. goto error;
  1341. if (xfs_alloc_delwrite_queue(btp))
  1342. goto error;
  1343. xfs_alloc_bufhash(btp, external);
  1344. return btp;
  1345. error:
  1346. kmem_free(btp);
  1347. return NULL;
  1348. }
  1349. /*
  1350. * Delayed write buffer handling
  1351. */
  1352. STATIC void
  1353. xfs_buf_delwri_queue(
  1354. xfs_buf_t *bp,
  1355. int unlock)
  1356. {
  1357. struct list_head *dwq = &bp->b_target->bt_delwrite_queue;
  1358. spinlock_t *dwlk = &bp->b_target->bt_delwrite_lock;
  1359. trace_xfs_buf_delwri_queue(bp, _RET_IP_);
  1360. ASSERT((bp->b_flags&(XBF_DELWRI|XBF_ASYNC)) == (XBF_DELWRI|XBF_ASYNC));
  1361. spin_lock(dwlk);
  1362. /* If already in the queue, dequeue and place at tail */
  1363. if (!list_empty(&bp->b_list)) {
  1364. ASSERT(bp->b_flags & _XBF_DELWRI_Q);
  1365. if (unlock)
  1366. atomic_dec(&bp->b_hold);
  1367. list_del(&bp->b_list);
  1368. }
  1369. bp->b_flags |= _XBF_DELWRI_Q;
  1370. list_add_tail(&bp->b_list, dwq);
  1371. bp->b_queuetime = jiffies;
  1372. spin_unlock(dwlk);
  1373. if (unlock)
  1374. xfs_buf_unlock(bp);
  1375. }
  1376. void
  1377. xfs_buf_delwri_dequeue(
  1378. xfs_buf_t *bp)
  1379. {
  1380. spinlock_t *dwlk = &bp->b_target->bt_delwrite_lock;
  1381. int dequeued = 0;
  1382. spin_lock(dwlk);
  1383. if ((bp->b_flags & XBF_DELWRI) && !list_empty(&bp->b_list)) {
  1384. ASSERT(bp->b_flags & _XBF_DELWRI_Q);
  1385. list_del_init(&bp->b_list);
  1386. dequeued = 1;
  1387. }
  1388. bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
  1389. spin_unlock(dwlk);
  1390. if (dequeued)
  1391. xfs_buf_rele(bp);
  1392. trace_xfs_buf_delwri_dequeue(bp, _RET_IP_);
  1393. }
  1394. STATIC void
  1395. xfs_buf_runall_queues(
  1396. struct workqueue_struct *queue)
  1397. {
  1398. flush_workqueue(queue);
  1399. }
  1400. STATIC int
  1401. xfsbufd_wakeup(
  1402. int priority,
  1403. gfp_t mask)
  1404. {
  1405. xfs_buftarg_t *btp;
  1406. spin_lock(&xfs_buftarg_lock);
  1407. list_for_each_entry(btp, &xfs_buftarg_list, bt_list) {
  1408. if (test_bit(XBT_FORCE_SLEEP, &btp->bt_flags))
  1409. continue;
  1410. set_bit(XBT_FORCE_FLUSH, &btp->bt_flags);
  1411. wake_up_process(btp->bt_task);
  1412. }
  1413. spin_unlock(&xfs_buftarg_lock);
  1414. return 0;
  1415. }
  1416. /*
  1417. * Move as many buffers as specified to the supplied list
  1418. * idicating if we skipped any buffers to prevent deadlocks.
  1419. */
  1420. STATIC int
  1421. xfs_buf_delwri_split(
  1422. xfs_buftarg_t *target,
  1423. struct list_head *list,
  1424. unsigned long age)
  1425. {
  1426. xfs_buf_t *bp, *n;
  1427. struct list_head *dwq = &target->bt_delwrite_queue;
  1428. spinlock_t *dwlk = &target->bt_delwrite_lock;
  1429. int skipped = 0;
  1430. int force;
  1431. force = test_and_clear_bit(XBT_FORCE_FLUSH, &target->bt_flags);
  1432. INIT_LIST_HEAD(list);
  1433. spin_lock(dwlk);
  1434. list_for_each_entry_safe(bp, n, dwq, b_list) {
  1435. trace_xfs_buf_delwri_split(bp, _RET_IP_);
  1436. ASSERT(bp->b_flags & XBF_DELWRI);
  1437. if (!xfs_buf_ispin(bp) && !xfs_buf_cond_lock(bp)) {
  1438. if (!force &&
  1439. time_before(jiffies, bp->b_queuetime + age)) {
  1440. xfs_buf_unlock(bp);
  1441. break;
  1442. }
  1443. bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q|
  1444. _XBF_RUN_QUEUES);
  1445. bp->b_flags |= XBF_WRITE;
  1446. list_move_tail(&bp->b_list, list);
  1447. } else
  1448. skipped++;
  1449. }
  1450. spin_unlock(dwlk);
  1451. return skipped;
  1452. }
  1453. STATIC int
  1454. xfsbufd(
  1455. void *data)
  1456. {
  1457. struct list_head tmp;
  1458. xfs_buftarg_t *target = (xfs_buftarg_t *)data;
  1459. int count;
  1460. xfs_buf_t *bp;
  1461. current->flags |= PF_MEMALLOC;
  1462. set_freezable();
  1463. do {
  1464. if (unlikely(freezing(current))) {
  1465. set_bit(XBT_FORCE_SLEEP, &target->bt_flags);
  1466. refrigerator();
  1467. } else {
  1468. clear_bit(XBT_FORCE_SLEEP, &target->bt_flags);
  1469. }
  1470. schedule_timeout_interruptible(
  1471. xfs_buf_timer_centisecs * msecs_to_jiffies(10));
  1472. xfs_buf_delwri_split(target, &tmp,
  1473. xfs_buf_age_centisecs * msecs_to_jiffies(10));
  1474. count = 0;
  1475. while (!list_empty(&tmp)) {
  1476. bp = list_entry(tmp.next, xfs_buf_t, b_list);
  1477. ASSERT(target == bp->b_target);
  1478. list_del_init(&bp->b_list);
  1479. xfs_buf_iostrategy(bp);
  1480. count++;
  1481. }
  1482. if (as_list_len > 0)
  1483. purge_addresses();
  1484. if (count)
  1485. blk_run_address_space(target->bt_mapping);
  1486. } while (!kthread_should_stop());
  1487. return 0;
  1488. }
  1489. /*
  1490. * Go through all incore buffers, and release buffers if they belong to
  1491. * the given device. This is used in filesystem error handling to
  1492. * preserve the consistency of its metadata.
  1493. */
  1494. int
  1495. xfs_flush_buftarg(
  1496. xfs_buftarg_t *target,
  1497. int wait)
  1498. {
  1499. struct list_head tmp;
  1500. xfs_buf_t *bp, *n;
  1501. int pincount = 0;
  1502. xfs_buf_runall_queues(xfsconvertd_workqueue);
  1503. xfs_buf_runall_queues(xfsdatad_workqueue);
  1504. xfs_buf_runall_queues(xfslogd_workqueue);
  1505. set_bit(XBT_FORCE_FLUSH, &target->bt_flags);
  1506. pincount = xfs_buf_delwri_split(target, &tmp, 0);
  1507. /*
  1508. * Dropped the delayed write list lock, now walk the temporary list
  1509. */
  1510. list_for_each_entry_safe(bp, n, &tmp, b_list) {
  1511. ASSERT(target == bp->b_target);
  1512. if (wait)
  1513. bp->b_flags &= ~XBF_ASYNC;
  1514. else
  1515. list_del_init(&bp->b_list);
  1516. xfs_buf_iostrategy(bp);
  1517. }
  1518. if (wait)
  1519. blk_run_address_space(target->bt_mapping);
  1520. /*
  1521. * Remaining list items must be flushed before returning
  1522. */
  1523. while (!list_empty(&tmp)) {
  1524. bp = list_entry(tmp.next, xfs_buf_t, b_list);
  1525. list_del_init(&bp->b_list);
  1526. xfs_iowait(bp);
  1527. xfs_buf_relse(bp);
  1528. }
  1529. return pincount;
  1530. }
  1531. int __init
  1532. xfs_buf_init(void)
  1533. {
  1534. xfs_buf_zone = kmem_zone_init_flags(sizeof(xfs_buf_t), "xfs_buf",
  1535. KM_ZONE_HWALIGN, NULL);
  1536. if (!xfs_buf_zone)
  1537. goto out;
  1538. xfslogd_workqueue = create_workqueue("xfslogd");
  1539. if (!xfslogd_workqueue)
  1540. goto out_free_buf_zone;
  1541. xfsdatad_workqueue = create_workqueue("xfsdatad");
  1542. if (!xfsdatad_workqueue)
  1543. goto out_destroy_xfslogd_workqueue;
  1544. xfsconvertd_workqueue = create_workqueue("xfsconvertd");
  1545. if (!xfsconvertd_workqueue)
  1546. goto out_destroy_xfsdatad_workqueue;
  1547. register_shrinker(&xfs_buf_shake);
  1548. return 0;
  1549. out_destroy_xfsdatad_workqueue:
  1550. destroy_workqueue(xfsdatad_workqueue);
  1551. out_destroy_xfslogd_workqueue:
  1552. destroy_workqueue(xfslogd_workqueue);
  1553. out_free_buf_zone:
  1554. kmem_zone_destroy(xfs_buf_zone);
  1555. out:
  1556. return -ENOMEM;
  1557. }
  1558. void
  1559. xfs_buf_terminate(void)
  1560. {
  1561. unregister_shrinker(&xfs_buf_shake);
  1562. destroy_workqueue(xfsconvertd_workqueue);
  1563. destroy_workqueue(xfsdatad_workqueue);
  1564. destroy_workqueue(xfslogd_workqueue);
  1565. kmem_zone_destroy(xfs_buf_zone);
  1566. }
  1567. #ifdef CONFIG_KDB_MODULES
  1568. struct list_head *
  1569. xfs_get_buftarg_list(void)
  1570. {
  1571. return &xfs_buftarg_list;
  1572. }
  1573. #endif