xfs_buf.c 42 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877
  1. /*
  2. * Copyright (c) 2000-2006 Silicon Graphics, Inc.
  3. * All Rights Reserved.
  4. *
  5. * This program is free software; you can redistribute it and/or
  6. * modify it under the terms of the GNU General Public License as
  7. * published by the Free Software Foundation.
  8. *
  9. * This program is distributed in the hope that it would be useful,
  10. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. * GNU General Public License for more details.
  13. *
  14. * You should have received a copy of the GNU General Public License
  15. * along with this program; if not, write the Free Software Foundation,
  16. * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  17. */
  18. #include "xfs.h"
  19. #include <linux/stddef.h>
  20. #include <linux/errno.h>
  21. #include <linux/gfp.h>
  22. #include <linux/pagemap.h>
  23. #include <linux/init.h>
  24. #include <linux/vmalloc.h>
  25. #include <linux/bio.h>
  26. #include <linux/sysctl.h>
  27. #include <linux/proc_fs.h>
  28. #include <linux/workqueue.h>
  29. #include <linux/percpu.h>
  30. #include <linux/blkdev.h>
  31. #include <linux/hash.h>
  32. #include <linux/kthread.h>
  33. #include <linux/migrate.h>
  34. #include <linux/backing-dev.h>
  35. #include <linux/freezer.h>
  36. #include "xfs_sb.h"
  37. #include "xfs_inum.h"
  38. #include "xfs_log.h"
  39. #include "xfs_ag.h"
  40. #include "xfs_mount.h"
  41. #include "xfs_trace.h"
  42. static kmem_zone_t *xfs_buf_zone;
  43. STATIC int xfsbufd(void *);
  44. STATIC void xfs_buf_delwri_queue(xfs_buf_t *, int);
  45. static struct workqueue_struct *xfslogd_workqueue;
  46. struct workqueue_struct *xfsdatad_workqueue;
  47. struct workqueue_struct *xfsconvertd_workqueue;
  48. #ifdef XFS_BUF_LOCK_TRACKING
  49. # define XB_SET_OWNER(bp) ((bp)->b_last_holder = current->pid)
  50. # define XB_CLEAR_OWNER(bp) ((bp)->b_last_holder = -1)
  51. # define XB_GET_OWNER(bp) ((bp)->b_last_holder)
  52. #else
  53. # define XB_SET_OWNER(bp) do { } while (0)
  54. # define XB_CLEAR_OWNER(bp) do { } while (0)
  55. # define XB_GET_OWNER(bp) do { } while (0)
  56. #endif
  57. #define xb_to_gfp(flags) \
  58. ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : \
  59. ((flags) & XBF_DONT_BLOCK) ? GFP_NOFS : GFP_KERNEL) | __GFP_NOWARN)
  60. #define xb_to_km(flags) \
  61. (((flags) & XBF_DONT_BLOCK) ? KM_NOFS : KM_SLEEP)
  62. #define xfs_buf_allocate(flags) \
  63. kmem_zone_alloc(xfs_buf_zone, xb_to_km(flags))
  64. #define xfs_buf_deallocate(bp) \
  65. kmem_zone_free(xfs_buf_zone, (bp));
  66. static inline int
  67. xfs_buf_is_vmapped(
  68. struct xfs_buf *bp)
  69. {
  70. /*
  71. * Return true if the buffer is vmapped.
  72. *
  73. * The XBF_MAPPED flag is set if the buffer should be mapped, but the
  74. * code is clever enough to know it doesn't have to map a single page,
  75. * so the check has to be both for XBF_MAPPED and bp->b_page_count > 1.
  76. */
  77. return (bp->b_flags & XBF_MAPPED) && bp->b_page_count > 1;
  78. }
  79. static inline int
  80. xfs_buf_vmap_len(
  81. struct xfs_buf *bp)
  82. {
  83. return (bp->b_page_count * PAGE_SIZE) - bp->b_offset;
  84. }
  85. /*
  86. * xfs_buf_lru_add - add a buffer to the LRU.
  87. *
  88. * The LRU takes a new reference to the buffer so that it will only be freed
  89. * once the shrinker takes the buffer off the LRU.
  90. */
  91. STATIC void
  92. xfs_buf_lru_add(
  93. struct xfs_buf *bp)
  94. {
  95. struct xfs_buftarg *btp = bp->b_target;
  96. spin_lock(&btp->bt_lru_lock);
  97. if (list_empty(&bp->b_lru)) {
  98. atomic_inc(&bp->b_hold);
  99. list_add_tail(&bp->b_lru, &btp->bt_lru);
  100. btp->bt_lru_nr++;
  101. }
  102. spin_unlock(&btp->bt_lru_lock);
  103. }
  104. /*
  105. * xfs_buf_lru_del - remove a buffer from the LRU
  106. *
  107. * The unlocked check is safe here because it only occurs when there are not
  108. * b_lru_ref counts left on the inode under the pag->pag_buf_lock. it is there
  109. * to optimise the shrinker removing the buffer from the LRU and calling
  110. * xfs_buf_free(). i.e. it removes an unnecessary round trip on the
  111. * bt_lru_lock.
  112. */
  113. STATIC void
  114. xfs_buf_lru_del(
  115. struct xfs_buf *bp)
  116. {
  117. struct xfs_buftarg *btp = bp->b_target;
  118. if (list_empty(&bp->b_lru))
  119. return;
  120. spin_lock(&btp->bt_lru_lock);
  121. if (!list_empty(&bp->b_lru)) {
  122. list_del_init(&bp->b_lru);
  123. btp->bt_lru_nr--;
  124. }
  125. spin_unlock(&btp->bt_lru_lock);
  126. }
  127. /*
  128. * When we mark a buffer stale, we remove the buffer from the LRU and clear the
  129. * b_lru_ref count so that the buffer is freed immediately when the buffer
  130. * reference count falls to zero. If the buffer is already on the LRU, we need
  131. * to remove the reference that LRU holds on the buffer.
  132. *
  133. * This prevents build-up of stale buffers on the LRU.
  134. */
  135. void
  136. xfs_buf_stale(
  137. struct xfs_buf *bp)
  138. {
  139. bp->b_flags |= XBF_STALE;
  140. atomic_set(&(bp)->b_lru_ref, 0);
  141. if (!list_empty(&bp->b_lru)) {
  142. struct xfs_buftarg *btp = bp->b_target;
  143. spin_lock(&btp->bt_lru_lock);
  144. if (!list_empty(&bp->b_lru)) {
  145. list_del_init(&bp->b_lru);
  146. btp->bt_lru_nr--;
  147. atomic_dec(&bp->b_hold);
  148. }
  149. spin_unlock(&btp->bt_lru_lock);
  150. }
  151. ASSERT(atomic_read(&bp->b_hold) >= 1);
  152. }
  153. STATIC void
  154. _xfs_buf_initialize(
  155. xfs_buf_t *bp,
  156. xfs_buftarg_t *target,
  157. xfs_off_t range_base,
  158. size_t range_length,
  159. xfs_buf_flags_t flags)
  160. {
  161. /*
  162. * We don't want certain flags to appear in b_flags.
  163. */
  164. flags &= ~(XBF_LOCK|XBF_MAPPED|XBF_DONT_BLOCK|XBF_READ_AHEAD);
  165. memset(bp, 0, sizeof(xfs_buf_t));
  166. atomic_set(&bp->b_hold, 1);
  167. atomic_set(&bp->b_lru_ref, 1);
  168. init_completion(&bp->b_iowait);
  169. INIT_LIST_HEAD(&bp->b_lru);
  170. INIT_LIST_HEAD(&bp->b_list);
  171. RB_CLEAR_NODE(&bp->b_rbnode);
  172. sema_init(&bp->b_sema, 0); /* held, no waiters */
  173. XB_SET_OWNER(bp);
  174. bp->b_target = target;
  175. bp->b_file_offset = range_base;
  176. /*
  177. * Set buffer_length and count_desired to the same value initially.
  178. * I/O routines should use count_desired, which will be the same in
  179. * most cases but may be reset (e.g. XFS recovery).
  180. */
  181. bp->b_buffer_length = bp->b_count_desired = range_length;
  182. bp->b_flags = flags;
  183. bp->b_bn = XFS_BUF_DADDR_NULL;
  184. atomic_set(&bp->b_pin_count, 0);
  185. init_waitqueue_head(&bp->b_waiters);
  186. XFS_STATS_INC(xb_create);
  187. trace_xfs_buf_init(bp, _RET_IP_);
  188. }
  189. /*
  190. * Allocate a page array capable of holding a specified number
  191. * of pages, and point the page buf at it.
  192. */
  193. STATIC int
  194. _xfs_buf_get_pages(
  195. xfs_buf_t *bp,
  196. int page_count,
  197. xfs_buf_flags_t flags)
  198. {
  199. /* Make sure that we have a page list */
  200. if (bp->b_pages == NULL) {
  201. bp->b_offset = xfs_buf_poff(bp->b_file_offset);
  202. bp->b_page_count = page_count;
  203. if (page_count <= XB_PAGES) {
  204. bp->b_pages = bp->b_page_array;
  205. } else {
  206. bp->b_pages = kmem_alloc(sizeof(struct page *) *
  207. page_count, xb_to_km(flags));
  208. if (bp->b_pages == NULL)
  209. return -ENOMEM;
  210. }
  211. memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
  212. }
  213. return 0;
  214. }
  215. /*
  216. * Frees b_pages if it was allocated.
  217. */
  218. STATIC void
  219. _xfs_buf_free_pages(
  220. xfs_buf_t *bp)
  221. {
  222. if (bp->b_pages != bp->b_page_array) {
  223. kmem_free(bp->b_pages);
  224. bp->b_pages = NULL;
  225. }
  226. }
  227. /*
  228. * Releases the specified buffer.
  229. *
  230. * The modification state of any associated pages is left unchanged.
  231. * The buffer most not be on any hash - use xfs_buf_rele instead for
  232. * hashed and refcounted buffers
  233. */
  234. void
  235. xfs_buf_free(
  236. xfs_buf_t *bp)
  237. {
  238. trace_xfs_buf_free(bp, _RET_IP_);
  239. ASSERT(list_empty(&bp->b_lru));
  240. if (bp->b_flags & _XBF_PAGES) {
  241. uint i;
  242. if (xfs_buf_is_vmapped(bp))
  243. vm_unmap_ram(bp->b_addr - bp->b_offset,
  244. bp->b_page_count);
  245. for (i = 0; i < bp->b_page_count; i++) {
  246. struct page *page = bp->b_pages[i];
  247. __free_page(page);
  248. }
  249. } else if (bp->b_flags & _XBF_KMEM)
  250. kmem_free(bp->b_addr);
  251. _xfs_buf_free_pages(bp);
  252. xfs_buf_deallocate(bp);
  253. }
  254. /*
  255. * Allocates all the pages for buffer in question and builds it's page list.
  256. */
  257. STATIC int
  258. xfs_buf_allocate_memory(
  259. xfs_buf_t *bp,
  260. uint flags)
  261. {
  262. size_t size = bp->b_count_desired;
  263. size_t nbytes, offset;
  264. gfp_t gfp_mask = xb_to_gfp(flags);
  265. unsigned short page_count, i;
  266. xfs_off_t end;
  267. int error;
  268. /*
  269. * for buffers that are contained within a single page, just allocate
  270. * the memory from the heap - there's no need for the complexity of
  271. * page arrays to keep allocation down to order 0.
  272. */
  273. if (bp->b_buffer_length < PAGE_SIZE) {
  274. bp->b_addr = kmem_alloc(bp->b_buffer_length, xb_to_km(flags));
  275. if (!bp->b_addr) {
  276. /* low memory - use alloc_page loop instead */
  277. goto use_alloc_page;
  278. }
  279. if (((unsigned long)(bp->b_addr + bp->b_buffer_length - 1) &
  280. PAGE_MASK) !=
  281. ((unsigned long)bp->b_addr & PAGE_MASK)) {
  282. /* b_addr spans two pages - use alloc_page instead */
  283. kmem_free(bp->b_addr);
  284. bp->b_addr = NULL;
  285. goto use_alloc_page;
  286. }
  287. bp->b_offset = offset_in_page(bp->b_addr);
  288. bp->b_pages = bp->b_page_array;
  289. bp->b_pages[0] = virt_to_page(bp->b_addr);
  290. bp->b_page_count = 1;
  291. bp->b_flags |= XBF_MAPPED | _XBF_KMEM;
  292. return 0;
  293. }
  294. use_alloc_page:
  295. end = bp->b_file_offset + bp->b_buffer_length;
  296. page_count = xfs_buf_btoc(end) - xfs_buf_btoct(bp->b_file_offset);
  297. error = _xfs_buf_get_pages(bp, page_count, flags);
  298. if (unlikely(error))
  299. return error;
  300. offset = bp->b_offset;
  301. bp->b_flags |= _XBF_PAGES;
  302. for (i = 0; i < bp->b_page_count; i++) {
  303. struct page *page;
  304. uint retries = 0;
  305. retry:
  306. page = alloc_page(gfp_mask);
  307. if (unlikely(page == NULL)) {
  308. if (flags & XBF_READ_AHEAD) {
  309. bp->b_page_count = i;
  310. error = ENOMEM;
  311. goto out_free_pages;
  312. }
  313. /*
  314. * This could deadlock.
  315. *
  316. * But until all the XFS lowlevel code is revamped to
  317. * handle buffer allocation failures we can't do much.
  318. */
  319. if (!(++retries % 100))
  320. xfs_err(NULL,
  321. "possible memory allocation deadlock in %s (mode:0x%x)",
  322. __func__, gfp_mask);
  323. XFS_STATS_INC(xb_page_retries);
  324. congestion_wait(BLK_RW_ASYNC, HZ/50);
  325. goto retry;
  326. }
  327. XFS_STATS_INC(xb_page_found);
  328. nbytes = min_t(size_t, size, PAGE_SIZE - offset);
  329. size -= nbytes;
  330. bp->b_pages[i] = page;
  331. offset = 0;
  332. }
  333. return 0;
  334. out_free_pages:
  335. for (i = 0; i < bp->b_page_count; i++)
  336. __free_page(bp->b_pages[i]);
  337. return error;
  338. }
  339. /*
  340. * Map buffer into kernel address-space if necessary.
  341. */
  342. STATIC int
  343. _xfs_buf_map_pages(
  344. xfs_buf_t *bp,
  345. uint flags)
  346. {
  347. ASSERT(bp->b_flags & _XBF_PAGES);
  348. if (bp->b_page_count == 1) {
  349. /* A single page buffer is always mappable */
  350. bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
  351. bp->b_flags |= XBF_MAPPED;
  352. } else if (flags & XBF_MAPPED) {
  353. int retried = 0;
  354. do {
  355. bp->b_addr = vm_map_ram(bp->b_pages, bp->b_page_count,
  356. -1, PAGE_KERNEL);
  357. if (bp->b_addr)
  358. break;
  359. vm_unmap_aliases();
  360. } while (retried++ <= 1);
  361. if (!bp->b_addr)
  362. return -ENOMEM;
  363. bp->b_addr += bp->b_offset;
  364. bp->b_flags |= XBF_MAPPED;
  365. }
  366. return 0;
  367. }
  368. /*
  369. * Finding and Reading Buffers
  370. */
  371. /*
  372. * Look up, and creates if absent, a lockable buffer for
  373. * a given range of an inode. The buffer is returned
  374. * locked. If other overlapping buffers exist, they are
  375. * released before the new buffer is created and locked,
  376. * which may imply that this call will block until those buffers
  377. * are unlocked. No I/O is implied by this call.
  378. */
  379. xfs_buf_t *
  380. _xfs_buf_find(
  381. xfs_buftarg_t *btp, /* block device target */
  382. xfs_off_t ioff, /* starting offset of range */
  383. size_t isize, /* length of range */
  384. xfs_buf_flags_t flags,
  385. xfs_buf_t *new_bp)
  386. {
  387. xfs_off_t range_base;
  388. size_t range_length;
  389. struct xfs_perag *pag;
  390. struct rb_node **rbp;
  391. struct rb_node *parent;
  392. xfs_buf_t *bp;
  393. range_base = (ioff << BBSHIFT);
  394. range_length = (isize << BBSHIFT);
  395. /* Check for IOs smaller than the sector size / not sector aligned */
  396. ASSERT(!(range_length < (1 << btp->bt_sshift)));
  397. ASSERT(!(range_base & (xfs_off_t)btp->bt_smask));
  398. /* get tree root */
  399. pag = xfs_perag_get(btp->bt_mount,
  400. xfs_daddr_to_agno(btp->bt_mount, ioff));
  401. /* walk tree */
  402. spin_lock(&pag->pag_buf_lock);
  403. rbp = &pag->pag_buf_tree.rb_node;
  404. parent = NULL;
  405. bp = NULL;
  406. while (*rbp) {
  407. parent = *rbp;
  408. bp = rb_entry(parent, struct xfs_buf, b_rbnode);
  409. if (range_base < bp->b_file_offset)
  410. rbp = &(*rbp)->rb_left;
  411. else if (range_base > bp->b_file_offset)
  412. rbp = &(*rbp)->rb_right;
  413. else {
  414. /*
  415. * found a block offset match. If the range doesn't
  416. * match, the only way this is allowed is if the buffer
  417. * in the cache is stale and the transaction that made
  418. * it stale has not yet committed. i.e. we are
  419. * reallocating a busy extent. Skip this buffer and
  420. * continue searching to the right for an exact match.
  421. */
  422. if (bp->b_buffer_length != range_length) {
  423. ASSERT(bp->b_flags & XBF_STALE);
  424. rbp = &(*rbp)->rb_right;
  425. continue;
  426. }
  427. atomic_inc(&bp->b_hold);
  428. goto found;
  429. }
  430. }
  431. /* No match found */
  432. if (new_bp) {
  433. _xfs_buf_initialize(new_bp, btp, range_base,
  434. range_length, flags);
  435. rb_link_node(&new_bp->b_rbnode, parent, rbp);
  436. rb_insert_color(&new_bp->b_rbnode, &pag->pag_buf_tree);
  437. /* the buffer keeps the perag reference until it is freed */
  438. new_bp->b_pag = pag;
  439. spin_unlock(&pag->pag_buf_lock);
  440. } else {
  441. XFS_STATS_INC(xb_miss_locked);
  442. spin_unlock(&pag->pag_buf_lock);
  443. xfs_perag_put(pag);
  444. }
  445. return new_bp;
  446. found:
  447. spin_unlock(&pag->pag_buf_lock);
  448. xfs_perag_put(pag);
  449. if (!xfs_buf_trylock(bp)) {
  450. if (flags & XBF_TRYLOCK) {
  451. xfs_buf_rele(bp);
  452. XFS_STATS_INC(xb_busy_locked);
  453. return NULL;
  454. }
  455. xfs_buf_lock(bp);
  456. XFS_STATS_INC(xb_get_locked_waited);
  457. }
  458. /*
  459. * if the buffer is stale, clear all the external state associated with
  460. * it. We need to keep flags such as how we allocated the buffer memory
  461. * intact here.
  462. */
  463. if (bp->b_flags & XBF_STALE) {
  464. ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
  465. bp->b_flags &= XBF_MAPPED | _XBF_KMEM | _XBF_PAGES;
  466. }
  467. trace_xfs_buf_find(bp, flags, _RET_IP_);
  468. XFS_STATS_INC(xb_get_locked);
  469. return bp;
  470. }
  471. /*
  472. * Assembles a buffer covering the specified range.
  473. * Storage in memory for all portions of the buffer will be allocated,
  474. * although backing storage may not be.
  475. */
  476. xfs_buf_t *
  477. xfs_buf_get(
  478. xfs_buftarg_t *target,/* target for buffer */
  479. xfs_off_t ioff, /* starting offset of range */
  480. size_t isize, /* length of range */
  481. xfs_buf_flags_t flags)
  482. {
  483. xfs_buf_t *bp, *new_bp;
  484. int error = 0;
  485. new_bp = xfs_buf_allocate(flags);
  486. if (unlikely(!new_bp))
  487. return NULL;
  488. bp = _xfs_buf_find(target, ioff, isize, flags, new_bp);
  489. if (bp == new_bp) {
  490. error = xfs_buf_allocate_memory(bp, flags);
  491. if (error)
  492. goto no_buffer;
  493. } else {
  494. xfs_buf_deallocate(new_bp);
  495. if (unlikely(bp == NULL))
  496. return NULL;
  497. }
  498. if (!(bp->b_flags & XBF_MAPPED)) {
  499. error = _xfs_buf_map_pages(bp, flags);
  500. if (unlikely(error)) {
  501. xfs_warn(target->bt_mount,
  502. "%s: failed to map pages\n", __func__);
  503. goto no_buffer;
  504. }
  505. }
  506. XFS_STATS_INC(xb_get);
  507. /*
  508. * Always fill in the block number now, the mapped cases can do
  509. * their own overlay of this later.
  510. */
  511. bp->b_bn = ioff;
  512. bp->b_count_desired = bp->b_buffer_length;
  513. trace_xfs_buf_get(bp, flags, _RET_IP_);
  514. return bp;
  515. no_buffer:
  516. if (flags & (XBF_LOCK | XBF_TRYLOCK))
  517. xfs_buf_unlock(bp);
  518. xfs_buf_rele(bp);
  519. return NULL;
  520. }
  521. STATIC int
  522. _xfs_buf_read(
  523. xfs_buf_t *bp,
  524. xfs_buf_flags_t flags)
  525. {
  526. int status;
  527. ASSERT(!(flags & (XBF_DELWRI|XBF_WRITE)));
  528. ASSERT(bp->b_bn != XFS_BUF_DADDR_NULL);
  529. bp->b_flags &= ~(XBF_WRITE | XBF_ASYNC | XBF_DELWRI | XBF_READ_AHEAD);
  530. bp->b_flags |= flags & (XBF_READ | XBF_ASYNC | XBF_READ_AHEAD);
  531. status = xfs_buf_iorequest(bp);
  532. if (status || XFS_BUF_ISERROR(bp) || (flags & XBF_ASYNC))
  533. return status;
  534. return xfs_buf_iowait(bp);
  535. }
  536. xfs_buf_t *
  537. xfs_buf_read(
  538. xfs_buftarg_t *target,
  539. xfs_off_t ioff,
  540. size_t isize,
  541. xfs_buf_flags_t flags)
  542. {
  543. xfs_buf_t *bp;
  544. flags |= XBF_READ;
  545. bp = xfs_buf_get(target, ioff, isize, flags);
  546. if (bp) {
  547. trace_xfs_buf_read(bp, flags, _RET_IP_);
  548. if (!XFS_BUF_ISDONE(bp)) {
  549. XFS_STATS_INC(xb_get_read);
  550. _xfs_buf_read(bp, flags);
  551. } else if (flags & XBF_ASYNC) {
  552. /*
  553. * Read ahead call which is already satisfied,
  554. * drop the buffer
  555. */
  556. goto no_buffer;
  557. } else {
  558. /* We do not want read in the flags */
  559. bp->b_flags &= ~XBF_READ;
  560. }
  561. }
  562. return bp;
  563. no_buffer:
  564. if (flags & (XBF_LOCK | XBF_TRYLOCK))
  565. xfs_buf_unlock(bp);
  566. xfs_buf_rele(bp);
  567. return NULL;
  568. }
  569. /*
  570. * If we are not low on memory then do the readahead in a deadlock
  571. * safe manner.
  572. */
  573. void
  574. xfs_buf_readahead(
  575. xfs_buftarg_t *target,
  576. xfs_off_t ioff,
  577. size_t isize)
  578. {
  579. if (bdi_read_congested(target->bt_bdi))
  580. return;
  581. xfs_buf_read(target, ioff, isize,
  582. XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD|XBF_DONT_BLOCK);
  583. }
  584. /*
  585. * Read an uncached buffer from disk. Allocates and returns a locked
  586. * buffer containing the disk contents or nothing.
  587. */
  588. struct xfs_buf *
  589. xfs_buf_read_uncached(
  590. struct xfs_mount *mp,
  591. struct xfs_buftarg *target,
  592. xfs_daddr_t daddr,
  593. size_t length,
  594. int flags)
  595. {
  596. xfs_buf_t *bp;
  597. int error;
  598. bp = xfs_buf_get_uncached(target, length, flags);
  599. if (!bp)
  600. return NULL;
  601. /* set up the buffer for a read IO */
  602. XFS_BUF_SET_ADDR(bp, daddr);
  603. XFS_BUF_READ(bp);
  604. XFS_BUF_BUSY(bp);
  605. xfsbdstrat(mp, bp);
  606. error = xfs_buf_iowait(bp);
  607. if (error || bp->b_error) {
  608. xfs_buf_relse(bp);
  609. return NULL;
  610. }
  611. return bp;
  612. }
  613. xfs_buf_t *
  614. xfs_buf_get_empty(
  615. size_t len,
  616. xfs_buftarg_t *target)
  617. {
  618. xfs_buf_t *bp;
  619. bp = xfs_buf_allocate(0);
  620. if (bp)
  621. _xfs_buf_initialize(bp, target, 0, len, 0);
  622. return bp;
  623. }
  624. /*
  625. * Return a buffer allocated as an empty buffer and associated to external
  626. * memory via xfs_buf_associate_memory() back to it's empty state.
  627. */
  628. void
  629. xfs_buf_set_empty(
  630. struct xfs_buf *bp,
  631. size_t len)
  632. {
  633. if (bp->b_pages)
  634. _xfs_buf_free_pages(bp);
  635. bp->b_pages = NULL;
  636. bp->b_page_count = 0;
  637. bp->b_addr = NULL;
  638. bp->b_file_offset = 0;
  639. bp->b_buffer_length = bp->b_count_desired = len;
  640. bp->b_bn = XFS_BUF_DADDR_NULL;
  641. bp->b_flags &= ~XBF_MAPPED;
  642. }
  643. static inline struct page *
  644. mem_to_page(
  645. void *addr)
  646. {
  647. if ((!is_vmalloc_addr(addr))) {
  648. return virt_to_page(addr);
  649. } else {
  650. return vmalloc_to_page(addr);
  651. }
  652. }
  653. int
  654. xfs_buf_associate_memory(
  655. xfs_buf_t *bp,
  656. void *mem,
  657. size_t len)
  658. {
  659. int rval;
  660. int i = 0;
  661. unsigned long pageaddr;
  662. unsigned long offset;
  663. size_t buflen;
  664. int page_count;
  665. pageaddr = (unsigned long)mem & PAGE_MASK;
  666. offset = (unsigned long)mem - pageaddr;
  667. buflen = PAGE_ALIGN(len + offset);
  668. page_count = buflen >> PAGE_SHIFT;
  669. /* Free any previous set of page pointers */
  670. if (bp->b_pages)
  671. _xfs_buf_free_pages(bp);
  672. bp->b_pages = NULL;
  673. bp->b_addr = mem;
  674. rval = _xfs_buf_get_pages(bp, page_count, XBF_DONT_BLOCK);
  675. if (rval)
  676. return rval;
  677. bp->b_offset = offset;
  678. for (i = 0; i < bp->b_page_count; i++) {
  679. bp->b_pages[i] = mem_to_page((void *)pageaddr);
  680. pageaddr += PAGE_SIZE;
  681. }
  682. bp->b_count_desired = len;
  683. bp->b_buffer_length = buflen;
  684. bp->b_flags |= XBF_MAPPED;
  685. return 0;
  686. }
  687. xfs_buf_t *
  688. xfs_buf_get_uncached(
  689. struct xfs_buftarg *target,
  690. size_t len,
  691. int flags)
  692. {
  693. unsigned long page_count = PAGE_ALIGN(len) >> PAGE_SHIFT;
  694. int error, i;
  695. xfs_buf_t *bp;
  696. bp = xfs_buf_allocate(0);
  697. if (unlikely(bp == NULL))
  698. goto fail;
  699. _xfs_buf_initialize(bp, target, 0, len, 0);
  700. error = _xfs_buf_get_pages(bp, page_count, 0);
  701. if (error)
  702. goto fail_free_buf;
  703. for (i = 0; i < page_count; i++) {
  704. bp->b_pages[i] = alloc_page(xb_to_gfp(flags));
  705. if (!bp->b_pages[i])
  706. goto fail_free_mem;
  707. }
  708. bp->b_flags |= _XBF_PAGES;
  709. error = _xfs_buf_map_pages(bp, XBF_MAPPED);
  710. if (unlikely(error)) {
  711. xfs_warn(target->bt_mount,
  712. "%s: failed to map pages\n", __func__);
  713. goto fail_free_mem;
  714. }
  715. trace_xfs_buf_get_uncached(bp, _RET_IP_);
  716. return bp;
  717. fail_free_mem:
  718. while (--i >= 0)
  719. __free_page(bp->b_pages[i]);
  720. _xfs_buf_free_pages(bp);
  721. fail_free_buf:
  722. xfs_buf_deallocate(bp);
  723. fail:
  724. return NULL;
  725. }
  726. /*
  727. * Increment reference count on buffer, to hold the buffer concurrently
  728. * with another thread which may release (free) the buffer asynchronously.
  729. * Must hold the buffer already to call this function.
  730. */
  731. void
  732. xfs_buf_hold(
  733. xfs_buf_t *bp)
  734. {
  735. trace_xfs_buf_hold(bp, _RET_IP_);
  736. atomic_inc(&bp->b_hold);
  737. }
  738. /*
  739. * Releases a hold on the specified buffer. If the
  740. * the hold count is 1, calls xfs_buf_free.
  741. */
  742. void
  743. xfs_buf_rele(
  744. xfs_buf_t *bp)
  745. {
  746. struct xfs_perag *pag = bp->b_pag;
  747. trace_xfs_buf_rele(bp, _RET_IP_);
  748. if (!pag) {
  749. ASSERT(list_empty(&bp->b_lru));
  750. ASSERT(RB_EMPTY_NODE(&bp->b_rbnode));
  751. if (atomic_dec_and_test(&bp->b_hold))
  752. xfs_buf_free(bp);
  753. return;
  754. }
  755. ASSERT(!RB_EMPTY_NODE(&bp->b_rbnode));
  756. ASSERT(atomic_read(&bp->b_hold) > 0);
  757. if (atomic_dec_and_lock(&bp->b_hold, &pag->pag_buf_lock)) {
  758. if (!(bp->b_flags & XBF_STALE) &&
  759. atomic_read(&bp->b_lru_ref)) {
  760. xfs_buf_lru_add(bp);
  761. spin_unlock(&pag->pag_buf_lock);
  762. } else {
  763. xfs_buf_lru_del(bp);
  764. ASSERT(!(bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)));
  765. rb_erase(&bp->b_rbnode, &pag->pag_buf_tree);
  766. spin_unlock(&pag->pag_buf_lock);
  767. xfs_perag_put(pag);
  768. xfs_buf_free(bp);
  769. }
  770. }
  771. }
  772. /*
  773. * Lock a buffer object, if it is not already locked.
  774. *
  775. * If we come across a stale, pinned, locked buffer, we know that we are
  776. * being asked to lock a buffer that has been reallocated. Because it is
  777. * pinned, we know that the log has not been pushed to disk and hence it
  778. * will still be locked. Rather than continuing to have trylock attempts
  779. * fail until someone else pushes the log, push it ourselves before
  780. * returning. This means that the xfsaild will not get stuck trying
  781. * to push on stale inode buffers.
  782. */
  783. int
  784. xfs_buf_trylock(
  785. struct xfs_buf *bp)
  786. {
  787. int locked;
  788. locked = down_trylock(&bp->b_sema) == 0;
  789. if (locked)
  790. XB_SET_OWNER(bp);
  791. else if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE))
  792. xfs_log_force(bp->b_target->bt_mount, 0);
  793. trace_xfs_buf_trylock(bp, _RET_IP_);
  794. return locked;
  795. }
  796. /*
  797. * Lock a buffer object.
  798. *
  799. * If we come across a stale, pinned, locked buffer, we know that we
  800. * are being asked to lock a buffer that has been reallocated. Because
  801. * it is pinned, we know that the log has not been pushed to disk and
  802. * hence it will still be locked. Rather than sleeping until someone
  803. * else pushes the log, push it ourselves before trying to get the lock.
  804. */
  805. void
  806. xfs_buf_lock(
  807. struct xfs_buf *bp)
  808. {
  809. trace_xfs_buf_lock(bp, _RET_IP_);
  810. if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE))
  811. xfs_log_force(bp->b_target->bt_mount, 0);
  812. down(&bp->b_sema);
  813. XB_SET_OWNER(bp);
  814. trace_xfs_buf_lock_done(bp, _RET_IP_);
  815. }
  816. /*
  817. * Releases the lock on the buffer object.
  818. * If the buffer is marked delwri but is not queued, do so before we
  819. * unlock the buffer as we need to set flags correctly. We also need to
  820. * take a reference for the delwri queue because the unlocker is going to
  821. * drop their's and they don't know we just queued it.
  822. */
  823. void
  824. xfs_buf_unlock(
  825. struct xfs_buf *bp)
  826. {
  827. if ((bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)) == XBF_DELWRI) {
  828. atomic_inc(&bp->b_hold);
  829. bp->b_flags |= XBF_ASYNC;
  830. xfs_buf_delwri_queue(bp, 0);
  831. }
  832. XB_CLEAR_OWNER(bp);
  833. up(&bp->b_sema);
  834. trace_xfs_buf_unlock(bp, _RET_IP_);
  835. }
  836. STATIC void
  837. xfs_buf_wait_unpin(
  838. xfs_buf_t *bp)
  839. {
  840. DECLARE_WAITQUEUE (wait, current);
  841. if (atomic_read(&bp->b_pin_count) == 0)
  842. return;
  843. add_wait_queue(&bp->b_waiters, &wait);
  844. for (;;) {
  845. set_current_state(TASK_UNINTERRUPTIBLE);
  846. if (atomic_read(&bp->b_pin_count) == 0)
  847. break;
  848. io_schedule();
  849. }
  850. remove_wait_queue(&bp->b_waiters, &wait);
  851. set_current_state(TASK_RUNNING);
  852. }
  853. /*
  854. * Buffer Utility Routines
  855. */
  856. STATIC void
  857. xfs_buf_iodone_work(
  858. struct work_struct *work)
  859. {
  860. xfs_buf_t *bp =
  861. container_of(work, xfs_buf_t, b_iodone_work);
  862. if (bp->b_iodone)
  863. (*(bp->b_iodone))(bp);
  864. else if (bp->b_flags & XBF_ASYNC)
  865. xfs_buf_relse(bp);
  866. }
  867. void
  868. xfs_buf_ioend(
  869. xfs_buf_t *bp,
  870. int schedule)
  871. {
  872. trace_xfs_buf_iodone(bp, _RET_IP_);
  873. bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
  874. if (bp->b_error == 0)
  875. bp->b_flags |= XBF_DONE;
  876. if ((bp->b_iodone) || (bp->b_flags & XBF_ASYNC)) {
  877. if (schedule) {
  878. INIT_WORK(&bp->b_iodone_work, xfs_buf_iodone_work);
  879. queue_work(xfslogd_workqueue, &bp->b_iodone_work);
  880. } else {
  881. xfs_buf_iodone_work(&bp->b_iodone_work);
  882. }
  883. } else {
  884. complete(&bp->b_iowait);
  885. }
  886. }
  887. void
  888. xfs_buf_ioerror(
  889. xfs_buf_t *bp,
  890. int error)
  891. {
  892. ASSERT(error >= 0 && error <= 0xffff);
  893. bp->b_error = (unsigned short)error;
  894. trace_xfs_buf_ioerror(bp, error, _RET_IP_);
  895. }
  896. int
  897. xfs_bwrite(
  898. struct xfs_mount *mp,
  899. struct xfs_buf *bp)
  900. {
  901. int error;
  902. bp->b_flags |= XBF_WRITE;
  903. bp->b_flags &= ~(XBF_ASYNC | XBF_READ);
  904. xfs_buf_delwri_dequeue(bp);
  905. xfs_bdstrat_cb(bp);
  906. error = xfs_buf_iowait(bp);
  907. if (error)
  908. xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
  909. xfs_buf_relse(bp);
  910. return error;
  911. }
  912. void
  913. xfs_bdwrite(
  914. void *mp,
  915. struct xfs_buf *bp)
  916. {
  917. trace_xfs_buf_bdwrite(bp, _RET_IP_);
  918. bp->b_flags &= ~XBF_READ;
  919. bp->b_flags |= (XBF_DELWRI | XBF_ASYNC);
  920. xfs_buf_delwri_queue(bp, 1);
  921. }
  922. /*
  923. * Called when we want to stop a buffer from getting written or read.
  924. * We attach the EIO error, muck with its flags, and call xfs_buf_ioend
  925. * so that the proper iodone callbacks get called.
  926. */
  927. STATIC int
  928. xfs_bioerror(
  929. xfs_buf_t *bp)
  930. {
  931. #ifdef XFSERRORDEBUG
  932. ASSERT(XFS_BUF_ISREAD(bp) || bp->b_iodone);
  933. #endif
  934. /*
  935. * No need to wait until the buffer is unpinned, we aren't flushing it.
  936. */
  937. XFS_BUF_ERROR(bp, EIO);
  938. /*
  939. * We're calling xfs_buf_ioend, so delete XBF_DONE flag.
  940. */
  941. XFS_BUF_UNREAD(bp);
  942. XFS_BUF_UNDELAYWRITE(bp);
  943. XFS_BUF_UNDONE(bp);
  944. XFS_BUF_STALE(bp);
  945. xfs_buf_ioend(bp, 0);
  946. return EIO;
  947. }
  948. /*
  949. * Same as xfs_bioerror, except that we are releasing the buffer
  950. * here ourselves, and avoiding the xfs_buf_ioend call.
  951. * This is meant for userdata errors; metadata bufs come with
  952. * iodone functions attached, so that we can track down errors.
  953. */
  954. STATIC int
  955. xfs_bioerror_relse(
  956. struct xfs_buf *bp)
  957. {
  958. int64_t fl = XFS_BUF_BFLAGS(bp);
  959. /*
  960. * No need to wait until the buffer is unpinned.
  961. * We aren't flushing it.
  962. *
  963. * chunkhold expects B_DONE to be set, whether
  964. * we actually finish the I/O or not. We don't want to
  965. * change that interface.
  966. */
  967. XFS_BUF_UNREAD(bp);
  968. XFS_BUF_UNDELAYWRITE(bp);
  969. XFS_BUF_DONE(bp);
  970. XFS_BUF_STALE(bp);
  971. bp->b_iodone = NULL;
  972. if (!(fl & XBF_ASYNC)) {
  973. /*
  974. * Mark b_error and B_ERROR _both_.
  975. * Lot's of chunkcache code assumes that.
  976. * There's no reason to mark error for
  977. * ASYNC buffers.
  978. */
  979. XFS_BUF_ERROR(bp, EIO);
  980. XFS_BUF_FINISH_IOWAIT(bp);
  981. } else {
  982. xfs_buf_relse(bp);
  983. }
  984. return EIO;
  985. }
  986. /*
  987. * All xfs metadata buffers except log state machine buffers
  988. * get this attached as their b_bdstrat callback function.
  989. * This is so that we can catch a buffer
  990. * after prematurely unpinning it to forcibly shutdown the filesystem.
  991. */
  992. int
  993. xfs_bdstrat_cb(
  994. struct xfs_buf *bp)
  995. {
  996. if (XFS_FORCED_SHUTDOWN(bp->b_target->bt_mount)) {
  997. trace_xfs_bdstrat_shut(bp, _RET_IP_);
  998. /*
  999. * Metadata write that didn't get logged but
  1000. * written delayed anyway. These aren't associated
  1001. * with a transaction, and can be ignored.
  1002. */
  1003. if (!bp->b_iodone && !XFS_BUF_ISREAD(bp))
  1004. return xfs_bioerror_relse(bp);
  1005. else
  1006. return xfs_bioerror(bp);
  1007. }
  1008. xfs_buf_iorequest(bp);
  1009. return 0;
  1010. }
  1011. /*
  1012. * Wrapper around bdstrat so that we can stop data from going to disk in case
  1013. * we are shutting down the filesystem. Typically user data goes thru this
  1014. * path; one of the exceptions is the superblock.
  1015. */
  1016. void
  1017. xfsbdstrat(
  1018. struct xfs_mount *mp,
  1019. struct xfs_buf *bp)
  1020. {
  1021. if (XFS_FORCED_SHUTDOWN(mp)) {
  1022. trace_xfs_bdstrat_shut(bp, _RET_IP_);
  1023. xfs_bioerror_relse(bp);
  1024. return;
  1025. }
  1026. xfs_buf_iorequest(bp);
  1027. }
  1028. STATIC void
  1029. _xfs_buf_ioend(
  1030. xfs_buf_t *bp,
  1031. int schedule)
  1032. {
  1033. if (atomic_dec_and_test(&bp->b_io_remaining) == 1)
  1034. xfs_buf_ioend(bp, schedule);
  1035. }
  1036. STATIC void
  1037. xfs_buf_bio_end_io(
  1038. struct bio *bio,
  1039. int error)
  1040. {
  1041. xfs_buf_t *bp = (xfs_buf_t *)bio->bi_private;
  1042. xfs_buf_ioerror(bp, -error);
  1043. if (!error && xfs_buf_is_vmapped(bp) && (bp->b_flags & XBF_READ))
  1044. invalidate_kernel_vmap_range(bp->b_addr, xfs_buf_vmap_len(bp));
  1045. _xfs_buf_ioend(bp, 1);
  1046. bio_put(bio);
  1047. }
  1048. STATIC void
  1049. _xfs_buf_ioapply(
  1050. xfs_buf_t *bp)
  1051. {
  1052. int rw, map_i, total_nr_pages, nr_pages;
  1053. struct bio *bio;
  1054. int offset = bp->b_offset;
  1055. int size = bp->b_count_desired;
  1056. sector_t sector = bp->b_bn;
  1057. total_nr_pages = bp->b_page_count;
  1058. map_i = 0;
  1059. if (bp->b_flags & XBF_WRITE) {
  1060. if (bp->b_flags & XBF_SYNCIO)
  1061. rw = WRITE_SYNC;
  1062. else
  1063. rw = WRITE;
  1064. if (bp->b_flags & XBF_FUA)
  1065. rw |= REQ_FUA;
  1066. if (bp->b_flags & XBF_FLUSH)
  1067. rw |= REQ_FLUSH;
  1068. } else if (bp->b_flags & XBF_READ_AHEAD) {
  1069. rw = READA;
  1070. } else {
  1071. rw = READ;
  1072. }
  1073. /* we only use the buffer cache for meta-data */
  1074. rw |= REQ_META;
  1075. next_chunk:
  1076. atomic_inc(&bp->b_io_remaining);
  1077. nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
  1078. if (nr_pages > total_nr_pages)
  1079. nr_pages = total_nr_pages;
  1080. bio = bio_alloc(GFP_NOIO, nr_pages);
  1081. bio->bi_bdev = bp->b_target->bt_bdev;
  1082. bio->bi_sector = sector;
  1083. bio->bi_end_io = xfs_buf_bio_end_io;
  1084. bio->bi_private = bp;
  1085. for (; size && nr_pages; nr_pages--, map_i++) {
  1086. int rbytes, nbytes = PAGE_SIZE - offset;
  1087. if (nbytes > size)
  1088. nbytes = size;
  1089. rbytes = bio_add_page(bio, bp->b_pages[map_i], nbytes, offset);
  1090. if (rbytes < nbytes)
  1091. break;
  1092. offset = 0;
  1093. sector += nbytes >> BBSHIFT;
  1094. size -= nbytes;
  1095. total_nr_pages--;
  1096. }
  1097. if (likely(bio->bi_size)) {
  1098. if (xfs_buf_is_vmapped(bp)) {
  1099. flush_kernel_vmap_range(bp->b_addr,
  1100. xfs_buf_vmap_len(bp));
  1101. }
  1102. submit_bio(rw, bio);
  1103. if (size)
  1104. goto next_chunk;
  1105. } else {
  1106. xfs_buf_ioerror(bp, EIO);
  1107. bio_put(bio);
  1108. }
  1109. }
  1110. int
  1111. xfs_buf_iorequest(
  1112. xfs_buf_t *bp)
  1113. {
  1114. trace_xfs_buf_iorequest(bp, _RET_IP_);
  1115. if (bp->b_flags & XBF_DELWRI) {
  1116. xfs_buf_delwri_queue(bp, 1);
  1117. return 0;
  1118. }
  1119. if (bp->b_flags & XBF_WRITE) {
  1120. xfs_buf_wait_unpin(bp);
  1121. }
  1122. xfs_buf_hold(bp);
  1123. /* Set the count to 1 initially, this will stop an I/O
  1124. * completion callout which happens before we have started
  1125. * all the I/O from calling xfs_buf_ioend too early.
  1126. */
  1127. atomic_set(&bp->b_io_remaining, 1);
  1128. _xfs_buf_ioapply(bp);
  1129. _xfs_buf_ioend(bp, 0);
  1130. xfs_buf_rele(bp);
  1131. return 0;
  1132. }
  1133. /*
  1134. * Waits for I/O to complete on the buffer supplied.
  1135. * It returns immediately if no I/O is pending.
  1136. * It returns the I/O error code, if any, or 0 if there was no error.
  1137. */
  1138. int
  1139. xfs_buf_iowait(
  1140. xfs_buf_t *bp)
  1141. {
  1142. trace_xfs_buf_iowait(bp, _RET_IP_);
  1143. wait_for_completion(&bp->b_iowait);
  1144. trace_xfs_buf_iowait_done(bp, _RET_IP_);
  1145. return bp->b_error;
  1146. }
  1147. xfs_caddr_t
  1148. xfs_buf_offset(
  1149. xfs_buf_t *bp,
  1150. size_t offset)
  1151. {
  1152. struct page *page;
  1153. if (bp->b_flags & XBF_MAPPED)
  1154. return XFS_BUF_PTR(bp) + offset;
  1155. offset += bp->b_offset;
  1156. page = bp->b_pages[offset >> PAGE_SHIFT];
  1157. return (xfs_caddr_t)page_address(page) + (offset & (PAGE_SIZE-1));
  1158. }
  1159. /*
  1160. * Move data into or out of a buffer.
  1161. */
  1162. void
  1163. xfs_buf_iomove(
  1164. xfs_buf_t *bp, /* buffer to process */
  1165. size_t boff, /* starting buffer offset */
  1166. size_t bsize, /* length to copy */
  1167. void *data, /* data address */
  1168. xfs_buf_rw_t mode) /* read/write/zero flag */
  1169. {
  1170. size_t bend, cpoff, csize;
  1171. struct page *page;
  1172. bend = boff + bsize;
  1173. while (boff < bend) {
  1174. page = bp->b_pages[xfs_buf_btoct(boff + bp->b_offset)];
  1175. cpoff = xfs_buf_poff(boff + bp->b_offset);
  1176. csize = min_t(size_t,
  1177. PAGE_SIZE-cpoff, bp->b_count_desired-boff);
  1178. ASSERT(((csize + cpoff) <= PAGE_SIZE));
  1179. switch (mode) {
  1180. case XBRW_ZERO:
  1181. memset(page_address(page) + cpoff, 0, csize);
  1182. break;
  1183. case XBRW_READ:
  1184. memcpy(data, page_address(page) + cpoff, csize);
  1185. break;
  1186. case XBRW_WRITE:
  1187. memcpy(page_address(page) + cpoff, data, csize);
  1188. }
  1189. boff += csize;
  1190. data += csize;
  1191. }
  1192. }
  1193. /*
  1194. * Handling of buffer targets (buftargs).
  1195. */
  1196. /*
  1197. * Wait for any bufs with callbacks that have been submitted but have not yet
  1198. * returned. These buffers will have an elevated hold count, so wait on those
  1199. * while freeing all the buffers only held by the LRU.
  1200. */
  1201. void
  1202. xfs_wait_buftarg(
  1203. struct xfs_buftarg *btp)
  1204. {
  1205. struct xfs_buf *bp;
  1206. restart:
  1207. spin_lock(&btp->bt_lru_lock);
  1208. while (!list_empty(&btp->bt_lru)) {
  1209. bp = list_first_entry(&btp->bt_lru, struct xfs_buf, b_lru);
  1210. if (atomic_read(&bp->b_hold) > 1) {
  1211. spin_unlock(&btp->bt_lru_lock);
  1212. delay(100);
  1213. goto restart;
  1214. }
  1215. /*
  1216. * clear the LRU reference count so the bufer doesn't get
  1217. * ignored in xfs_buf_rele().
  1218. */
  1219. atomic_set(&bp->b_lru_ref, 0);
  1220. spin_unlock(&btp->bt_lru_lock);
  1221. xfs_buf_rele(bp);
  1222. spin_lock(&btp->bt_lru_lock);
  1223. }
  1224. spin_unlock(&btp->bt_lru_lock);
  1225. }
  1226. int
  1227. xfs_buftarg_shrink(
  1228. struct shrinker *shrink,
  1229. struct shrink_control *sc)
  1230. {
  1231. struct xfs_buftarg *btp = container_of(shrink,
  1232. struct xfs_buftarg, bt_shrinker);
  1233. struct xfs_buf *bp;
  1234. int nr_to_scan = sc->nr_to_scan;
  1235. LIST_HEAD(dispose);
  1236. if (!nr_to_scan)
  1237. return btp->bt_lru_nr;
  1238. spin_lock(&btp->bt_lru_lock);
  1239. while (!list_empty(&btp->bt_lru)) {
  1240. if (nr_to_scan-- <= 0)
  1241. break;
  1242. bp = list_first_entry(&btp->bt_lru, struct xfs_buf, b_lru);
  1243. /*
  1244. * Decrement the b_lru_ref count unless the value is already
  1245. * zero. If the value is already zero, we need to reclaim the
  1246. * buffer, otherwise it gets another trip through the LRU.
  1247. */
  1248. if (!atomic_add_unless(&bp->b_lru_ref, -1, 0)) {
  1249. list_move_tail(&bp->b_lru, &btp->bt_lru);
  1250. continue;
  1251. }
  1252. /*
  1253. * remove the buffer from the LRU now to avoid needing another
  1254. * lock round trip inside xfs_buf_rele().
  1255. */
  1256. list_move(&bp->b_lru, &dispose);
  1257. btp->bt_lru_nr--;
  1258. }
  1259. spin_unlock(&btp->bt_lru_lock);
  1260. while (!list_empty(&dispose)) {
  1261. bp = list_first_entry(&dispose, struct xfs_buf, b_lru);
  1262. list_del_init(&bp->b_lru);
  1263. xfs_buf_rele(bp);
  1264. }
  1265. return btp->bt_lru_nr;
  1266. }
  1267. void
  1268. xfs_free_buftarg(
  1269. struct xfs_mount *mp,
  1270. struct xfs_buftarg *btp)
  1271. {
  1272. unregister_shrinker(&btp->bt_shrinker);
  1273. xfs_flush_buftarg(btp, 1);
  1274. if (mp->m_flags & XFS_MOUNT_BARRIER)
  1275. xfs_blkdev_issue_flush(btp);
  1276. kthread_stop(btp->bt_task);
  1277. kmem_free(btp);
  1278. }
  1279. STATIC int
  1280. xfs_setsize_buftarg_flags(
  1281. xfs_buftarg_t *btp,
  1282. unsigned int blocksize,
  1283. unsigned int sectorsize,
  1284. int verbose)
  1285. {
  1286. btp->bt_bsize = blocksize;
  1287. btp->bt_sshift = ffs(sectorsize) - 1;
  1288. btp->bt_smask = sectorsize - 1;
  1289. if (set_blocksize(btp->bt_bdev, sectorsize)) {
  1290. xfs_warn(btp->bt_mount,
  1291. "Cannot set_blocksize to %u on device %s\n",
  1292. sectorsize, XFS_BUFTARG_NAME(btp));
  1293. return EINVAL;
  1294. }
  1295. return 0;
  1296. }
  1297. /*
  1298. * When allocating the initial buffer target we have not yet
  1299. * read in the superblock, so don't know what sized sectors
  1300. * are being used is at this early stage. Play safe.
  1301. */
  1302. STATIC int
  1303. xfs_setsize_buftarg_early(
  1304. xfs_buftarg_t *btp,
  1305. struct block_device *bdev)
  1306. {
  1307. return xfs_setsize_buftarg_flags(btp,
  1308. PAGE_SIZE, bdev_logical_block_size(bdev), 0);
  1309. }
  1310. int
  1311. xfs_setsize_buftarg(
  1312. xfs_buftarg_t *btp,
  1313. unsigned int blocksize,
  1314. unsigned int sectorsize)
  1315. {
  1316. return xfs_setsize_buftarg_flags(btp, blocksize, sectorsize, 1);
  1317. }
  1318. STATIC int
  1319. xfs_alloc_delwrite_queue(
  1320. xfs_buftarg_t *btp,
  1321. const char *fsname)
  1322. {
  1323. INIT_LIST_HEAD(&btp->bt_delwrite_queue);
  1324. spin_lock_init(&btp->bt_delwrite_lock);
  1325. btp->bt_flags = 0;
  1326. btp->bt_task = kthread_run(xfsbufd, btp, "xfsbufd/%s", fsname);
  1327. if (IS_ERR(btp->bt_task))
  1328. return PTR_ERR(btp->bt_task);
  1329. return 0;
  1330. }
  1331. xfs_buftarg_t *
  1332. xfs_alloc_buftarg(
  1333. struct xfs_mount *mp,
  1334. struct block_device *bdev,
  1335. int external,
  1336. const char *fsname)
  1337. {
  1338. xfs_buftarg_t *btp;
  1339. btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
  1340. btp->bt_mount = mp;
  1341. btp->bt_dev = bdev->bd_dev;
  1342. btp->bt_bdev = bdev;
  1343. btp->bt_bdi = blk_get_backing_dev_info(bdev);
  1344. if (!btp->bt_bdi)
  1345. goto error;
  1346. INIT_LIST_HEAD(&btp->bt_lru);
  1347. spin_lock_init(&btp->bt_lru_lock);
  1348. if (xfs_setsize_buftarg_early(btp, bdev))
  1349. goto error;
  1350. if (xfs_alloc_delwrite_queue(btp, fsname))
  1351. goto error;
  1352. btp->bt_shrinker.shrink = xfs_buftarg_shrink;
  1353. btp->bt_shrinker.seeks = DEFAULT_SEEKS;
  1354. register_shrinker(&btp->bt_shrinker);
  1355. return btp;
  1356. error:
  1357. kmem_free(btp);
  1358. return NULL;
  1359. }
  1360. /*
  1361. * Delayed write buffer handling
  1362. */
  1363. STATIC void
  1364. xfs_buf_delwri_queue(
  1365. xfs_buf_t *bp,
  1366. int unlock)
  1367. {
  1368. struct list_head *dwq = &bp->b_target->bt_delwrite_queue;
  1369. spinlock_t *dwlk = &bp->b_target->bt_delwrite_lock;
  1370. trace_xfs_buf_delwri_queue(bp, _RET_IP_);
  1371. ASSERT((bp->b_flags&(XBF_DELWRI|XBF_ASYNC)) == (XBF_DELWRI|XBF_ASYNC));
  1372. spin_lock(dwlk);
  1373. /* If already in the queue, dequeue and place at tail */
  1374. if (!list_empty(&bp->b_list)) {
  1375. ASSERT(bp->b_flags & _XBF_DELWRI_Q);
  1376. if (unlock)
  1377. atomic_dec(&bp->b_hold);
  1378. list_del(&bp->b_list);
  1379. }
  1380. if (list_empty(dwq)) {
  1381. /* start xfsbufd as it is about to have something to do */
  1382. wake_up_process(bp->b_target->bt_task);
  1383. }
  1384. bp->b_flags |= _XBF_DELWRI_Q;
  1385. list_add_tail(&bp->b_list, dwq);
  1386. bp->b_queuetime = jiffies;
  1387. spin_unlock(dwlk);
  1388. if (unlock)
  1389. xfs_buf_unlock(bp);
  1390. }
  1391. void
  1392. xfs_buf_delwri_dequeue(
  1393. xfs_buf_t *bp)
  1394. {
  1395. spinlock_t *dwlk = &bp->b_target->bt_delwrite_lock;
  1396. int dequeued = 0;
  1397. spin_lock(dwlk);
  1398. if ((bp->b_flags & XBF_DELWRI) && !list_empty(&bp->b_list)) {
  1399. ASSERT(bp->b_flags & _XBF_DELWRI_Q);
  1400. list_del_init(&bp->b_list);
  1401. dequeued = 1;
  1402. }
  1403. bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
  1404. spin_unlock(dwlk);
  1405. if (dequeued)
  1406. xfs_buf_rele(bp);
  1407. trace_xfs_buf_delwri_dequeue(bp, _RET_IP_);
  1408. }
  1409. /*
  1410. * If a delwri buffer needs to be pushed before it has aged out, then promote
  1411. * it to the head of the delwri queue so that it will be flushed on the next
  1412. * xfsbufd run. We do this by resetting the queuetime of the buffer to be older
  1413. * than the age currently needed to flush the buffer. Hence the next time the
  1414. * xfsbufd sees it is guaranteed to be considered old enough to flush.
  1415. */
  1416. void
  1417. xfs_buf_delwri_promote(
  1418. struct xfs_buf *bp)
  1419. {
  1420. struct xfs_buftarg *btp = bp->b_target;
  1421. long age = xfs_buf_age_centisecs * msecs_to_jiffies(10) + 1;
  1422. ASSERT(bp->b_flags & XBF_DELWRI);
  1423. ASSERT(bp->b_flags & _XBF_DELWRI_Q);
  1424. /*
  1425. * Check the buffer age before locking the delayed write queue as we
  1426. * don't need to promote buffers that are already past the flush age.
  1427. */
  1428. if (bp->b_queuetime < jiffies - age)
  1429. return;
  1430. bp->b_queuetime = jiffies - age;
  1431. spin_lock(&btp->bt_delwrite_lock);
  1432. list_move(&bp->b_list, &btp->bt_delwrite_queue);
  1433. spin_unlock(&btp->bt_delwrite_lock);
  1434. }
  1435. STATIC void
  1436. xfs_buf_runall_queues(
  1437. struct workqueue_struct *queue)
  1438. {
  1439. flush_workqueue(queue);
  1440. }
  1441. /*
  1442. * Move as many buffers as specified to the supplied list
  1443. * idicating if we skipped any buffers to prevent deadlocks.
  1444. */
  1445. STATIC int
  1446. xfs_buf_delwri_split(
  1447. xfs_buftarg_t *target,
  1448. struct list_head *list,
  1449. unsigned long age)
  1450. {
  1451. xfs_buf_t *bp, *n;
  1452. struct list_head *dwq = &target->bt_delwrite_queue;
  1453. spinlock_t *dwlk = &target->bt_delwrite_lock;
  1454. int skipped = 0;
  1455. int force;
  1456. force = test_and_clear_bit(XBT_FORCE_FLUSH, &target->bt_flags);
  1457. INIT_LIST_HEAD(list);
  1458. spin_lock(dwlk);
  1459. list_for_each_entry_safe(bp, n, dwq, b_list) {
  1460. ASSERT(bp->b_flags & XBF_DELWRI);
  1461. if (!XFS_BUF_ISPINNED(bp) && xfs_buf_trylock(bp)) {
  1462. if (!force &&
  1463. time_before(jiffies, bp->b_queuetime + age)) {
  1464. xfs_buf_unlock(bp);
  1465. break;
  1466. }
  1467. bp->b_flags &= ~(XBF_DELWRI | _XBF_DELWRI_Q);
  1468. bp->b_flags |= XBF_WRITE;
  1469. list_move_tail(&bp->b_list, list);
  1470. trace_xfs_buf_delwri_split(bp, _RET_IP_);
  1471. } else
  1472. skipped++;
  1473. }
  1474. spin_unlock(dwlk);
  1475. return skipped;
  1476. }
  1477. /*
  1478. * Compare function is more complex than it needs to be because
  1479. * the return value is only 32 bits and we are doing comparisons
  1480. * on 64 bit values
  1481. */
  1482. static int
  1483. xfs_buf_cmp(
  1484. void *priv,
  1485. struct list_head *a,
  1486. struct list_head *b)
  1487. {
  1488. struct xfs_buf *ap = container_of(a, struct xfs_buf, b_list);
  1489. struct xfs_buf *bp = container_of(b, struct xfs_buf, b_list);
  1490. xfs_daddr_t diff;
  1491. diff = ap->b_bn - bp->b_bn;
  1492. if (diff < 0)
  1493. return -1;
  1494. if (diff > 0)
  1495. return 1;
  1496. return 0;
  1497. }
  1498. STATIC int
  1499. xfsbufd(
  1500. void *data)
  1501. {
  1502. xfs_buftarg_t *target = (xfs_buftarg_t *)data;
  1503. current->flags |= PF_MEMALLOC;
  1504. set_freezable();
  1505. do {
  1506. long age = xfs_buf_age_centisecs * msecs_to_jiffies(10);
  1507. long tout = xfs_buf_timer_centisecs * msecs_to_jiffies(10);
  1508. struct list_head tmp;
  1509. struct blk_plug plug;
  1510. if (unlikely(freezing(current))) {
  1511. set_bit(XBT_FORCE_SLEEP, &target->bt_flags);
  1512. refrigerator();
  1513. } else {
  1514. clear_bit(XBT_FORCE_SLEEP, &target->bt_flags);
  1515. }
  1516. /* sleep for a long time if there is nothing to do. */
  1517. if (list_empty(&target->bt_delwrite_queue))
  1518. tout = MAX_SCHEDULE_TIMEOUT;
  1519. schedule_timeout_interruptible(tout);
  1520. xfs_buf_delwri_split(target, &tmp, age);
  1521. list_sort(NULL, &tmp, xfs_buf_cmp);
  1522. blk_start_plug(&plug);
  1523. while (!list_empty(&tmp)) {
  1524. struct xfs_buf *bp;
  1525. bp = list_first_entry(&tmp, struct xfs_buf, b_list);
  1526. list_del_init(&bp->b_list);
  1527. xfs_bdstrat_cb(bp);
  1528. }
  1529. blk_finish_plug(&plug);
  1530. } while (!kthread_should_stop());
  1531. return 0;
  1532. }
  1533. /*
  1534. * Go through all incore buffers, and release buffers if they belong to
  1535. * the given device. This is used in filesystem error handling to
  1536. * preserve the consistency of its metadata.
  1537. */
  1538. int
  1539. xfs_flush_buftarg(
  1540. xfs_buftarg_t *target,
  1541. int wait)
  1542. {
  1543. xfs_buf_t *bp;
  1544. int pincount = 0;
  1545. LIST_HEAD(tmp_list);
  1546. LIST_HEAD(wait_list);
  1547. struct blk_plug plug;
  1548. xfs_buf_runall_queues(xfsconvertd_workqueue);
  1549. xfs_buf_runall_queues(xfsdatad_workqueue);
  1550. xfs_buf_runall_queues(xfslogd_workqueue);
  1551. set_bit(XBT_FORCE_FLUSH, &target->bt_flags);
  1552. pincount = xfs_buf_delwri_split(target, &tmp_list, 0);
  1553. /*
  1554. * Dropped the delayed write list lock, now walk the temporary list.
  1555. * All I/O is issued async and then if we need to wait for completion
  1556. * we do that after issuing all the IO.
  1557. */
  1558. list_sort(NULL, &tmp_list, xfs_buf_cmp);
  1559. blk_start_plug(&plug);
  1560. while (!list_empty(&tmp_list)) {
  1561. bp = list_first_entry(&tmp_list, struct xfs_buf, b_list);
  1562. ASSERT(target == bp->b_target);
  1563. list_del_init(&bp->b_list);
  1564. if (wait) {
  1565. bp->b_flags &= ~XBF_ASYNC;
  1566. list_add(&bp->b_list, &wait_list);
  1567. }
  1568. xfs_bdstrat_cb(bp);
  1569. }
  1570. blk_finish_plug(&plug);
  1571. if (wait) {
  1572. /* Wait for IO to complete. */
  1573. while (!list_empty(&wait_list)) {
  1574. bp = list_first_entry(&wait_list, struct xfs_buf, b_list);
  1575. list_del_init(&bp->b_list);
  1576. xfs_buf_iowait(bp);
  1577. xfs_buf_relse(bp);
  1578. }
  1579. }
  1580. return pincount;
  1581. }
  1582. int __init
  1583. xfs_buf_init(void)
  1584. {
  1585. xfs_buf_zone = kmem_zone_init_flags(sizeof(xfs_buf_t), "xfs_buf",
  1586. KM_ZONE_HWALIGN, NULL);
  1587. if (!xfs_buf_zone)
  1588. goto out;
  1589. xfslogd_workqueue = alloc_workqueue("xfslogd",
  1590. WQ_MEM_RECLAIM | WQ_HIGHPRI, 1);
  1591. if (!xfslogd_workqueue)
  1592. goto out_free_buf_zone;
  1593. xfsdatad_workqueue = alloc_workqueue("xfsdatad", WQ_MEM_RECLAIM, 1);
  1594. if (!xfsdatad_workqueue)
  1595. goto out_destroy_xfslogd_workqueue;
  1596. xfsconvertd_workqueue = alloc_workqueue("xfsconvertd",
  1597. WQ_MEM_RECLAIM, 1);
  1598. if (!xfsconvertd_workqueue)
  1599. goto out_destroy_xfsdatad_workqueue;
  1600. return 0;
  1601. out_destroy_xfsdatad_workqueue:
  1602. destroy_workqueue(xfsdatad_workqueue);
  1603. out_destroy_xfslogd_workqueue:
  1604. destroy_workqueue(xfslogd_workqueue);
  1605. out_free_buf_zone:
  1606. kmem_zone_destroy(xfs_buf_zone);
  1607. out:
  1608. return -ENOMEM;
  1609. }
  1610. void
  1611. xfs_buf_terminate(void)
  1612. {
  1613. destroy_workqueue(xfsconvertd_workqueue);
  1614. destroy_workqueue(xfsdatad_workqueue);
  1615. destroy_workqueue(xfslogd_workqueue);
  1616. kmem_zone_destroy(xfs_buf_zone);
  1617. }
  1618. #ifdef CONFIG_KDB_MODULES
  1619. struct list_head *
  1620. xfs_get_buftarg_list(void)
  1621. {
  1622. return &xfs_buftarg_list;
  1623. }
  1624. #endif