xfs_aops.c 38 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581
  1. /*
  2. * Copyright (c) 2000-2005 Silicon Graphics, Inc.
  3. * All Rights Reserved.
  4. *
  5. * This program is free software; you can redistribute it and/or
  6. * modify it under the terms of the GNU General Public License as
  7. * published by the Free Software Foundation.
  8. *
  9. * This program is distributed in the hope that it would be useful,
  10. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. * GNU General Public License for more details.
  13. *
  14. * You should have received a copy of the GNU General Public License
  15. * along with this program; if not, write the Free Software Foundation,
  16. * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  17. */
  18. #include "xfs.h"
  19. #include "xfs_bit.h"
  20. #include "xfs_log.h"
  21. #include "xfs_inum.h"
  22. #include "xfs_sb.h"
  23. #include "xfs_ag.h"
  24. #include "xfs_trans.h"
  25. #include "xfs_mount.h"
  26. #include "xfs_bmap_btree.h"
  27. #include "xfs_dinode.h"
  28. #include "xfs_inode.h"
  29. #include "xfs_alloc.h"
  30. #include "xfs_error.h"
  31. #include "xfs_rw.h"
  32. #include "xfs_iomap.h"
  33. #include "xfs_vnodeops.h"
  34. #include "xfs_trace.h"
  35. #include "xfs_bmap.h"
  36. #include <linux/gfp.h>
  37. #include <linux/mpage.h>
  38. #include <linux/pagevec.h>
  39. #include <linux/writeback.h>
  40. /*
  41. * Types of I/O for bmap clustering and I/O completion tracking.
  42. */
  43. enum {
  44. IO_READ, /* mapping for a read */
  45. IO_DELAY, /* mapping covers delalloc region */
  46. IO_UNWRITTEN, /* mapping covers allocated but uninitialized data */
  47. IO_NEW /* just allocated */
  48. };
  49. /*
  50. * Prime number of hash buckets since address is used as the key.
  51. */
  52. #define NVSYNC 37
  53. #define to_ioend_wq(v) (&xfs_ioend_wq[((unsigned long)v) % NVSYNC])
  54. static wait_queue_head_t xfs_ioend_wq[NVSYNC];
  55. void __init
  56. xfs_ioend_init(void)
  57. {
  58. int i;
  59. for (i = 0; i < NVSYNC; i++)
  60. init_waitqueue_head(&xfs_ioend_wq[i]);
  61. }
  62. void
  63. xfs_ioend_wait(
  64. xfs_inode_t *ip)
  65. {
  66. wait_queue_head_t *wq = to_ioend_wq(ip);
  67. wait_event(*wq, (atomic_read(&ip->i_iocount) == 0));
  68. }
  69. STATIC void
  70. xfs_ioend_wake(
  71. xfs_inode_t *ip)
  72. {
  73. if (atomic_dec_and_test(&ip->i_iocount))
  74. wake_up(to_ioend_wq(ip));
  75. }
  76. void
  77. xfs_count_page_state(
  78. struct page *page,
  79. int *delalloc,
  80. int *unwritten)
  81. {
  82. struct buffer_head *bh, *head;
  83. *delalloc = *unwritten = 0;
  84. bh = head = page_buffers(page);
  85. do {
  86. if (buffer_unwritten(bh))
  87. (*unwritten) = 1;
  88. else if (buffer_delay(bh))
  89. (*delalloc) = 1;
  90. } while ((bh = bh->b_this_page) != head);
  91. }
  92. STATIC struct block_device *
  93. xfs_find_bdev_for_inode(
  94. struct inode *inode)
  95. {
  96. struct xfs_inode *ip = XFS_I(inode);
  97. struct xfs_mount *mp = ip->i_mount;
  98. if (XFS_IS_REALTIME_INODE(ip))
  99. return mp->m_rtdev_targp->bt_bdev;
  100. else
  101. return mp->m_ddev_targp->bt_bdev;
  102. }
  103. /*
  104. * We're now finished for good with this ioend structure.
  105. * Update the page state via the associated buffer_heads,
  106. * release holds on the inode and bio, and finally free
  107. * up memory. Do not use the ioend after this.
  108. */
  109. STATIC void
  110. xfs_destroy_ioend(
  111. xfs_ioend_t *ioend)
  112. {
  113. struct buffer_head *bh, *next;
  114. struct xfs_inode *ip = XFS_I(ioend->io_inode);
  115. for (bh = ioend->io_buffer_head; bh; bh = next) {
  116. next = bh->b_private;
  117. bh->b_end_io(bh, !ioend->io_error);
  118. }
  119. /*
  120. * Volume managers supporting multiple paths can send back ENODEV
  121. * when the final path disappears. In this case continuing to fill
  122. * the page cache with dirty data which cannot be written out is
  123. * evil, so prevent that.
  124. */
  125. if (unlikely(ioend->io_error == -ENODEV)) {
  126. xfs_do_force_shutdown(ip->i_mount, SHUTDOWN_DEVICE_REQ,
  127. __FILE__, __LINE__);
  128. }
  129. xfs_ioend_wake(ip);
  130. mempool_free(ioend, xfs_ioend_pool);
  131. }
  132. /*
  133. * If the end of the current ioend is beyond the current EOF,
  134. * return the new EOF value, otherwise zero.
  135. */
  136. STATIC xfs_fsize_t
  137. xfs_ioend_new_eof(
  138. xfs_ioend_t *ioend)
  139. {
  140. xfs_inode_t *ip = XFS_I(ioend->io_inode);
  141. xfs_fsize_t isize;
  142. xfs_fsize_t bsize;
  143. bsize = ioend->io_offset + ioend->io_size;
  144. isize = MAX(ip->i_size, ip->i_new_size);
  145. isize = MIN(isize, bsize);
  146. return isize > ip->i_d.di_size ? isize : 0;
  147. }
  148. /*
  149. * Update on-disk file size now that data has been written to disk. The
  150. * current in-memory file size is i_size. If a write is beyond eof i_new_size
  151. * will be the intended file size until i_size is updated. If this write does
  152. * not extend all the way to the valid file size then restrict this update to
  153. * the end of the write.
  154. *
  155. * This function does not block as blocking on the inode lock in IO completion
  156. * can lead to IO completion order dependency deadlocks.. If it can't get the
  157. * inode ilock it will return EAGAIN. Callers must handle this.
  158. */
  159. STATIC int
  160. xfs_setfilesize(
  161. xfs_ioend_t *ioend)
  162. {
  163. xfs_inode_t *ip = XFS_I(ioend->io_inode);
  164. xfs_fsize_t isize;
  165. ASSERT((ip->i_d.di_mode & S_IFMT) == S_IFREG);
  166. ASSERT(ioend->io_type != IO_READ);
  167. if (unlikely(ioend->io_error))
  168. return 0;
  169. if (!xfs_ilock_nowait(ip, XFS_ILOCK_EXCL))
  170. return EAGAIN;
  171. isize = xfs_ioend_new_eof(ioend);
  172. if (isize) {
  173. ip->i_d.di_size = isize;
  174. xfs_mark_inode_dirty(ip);
  175. }
  176. xfs_iunlock(ip, XFS_ILOCK_EXCL);
  177. return 0;
  178. }
  179. /*
  180. * Schedule IO completion handling on the final put of an ioend.
  181. */
  182. STATIC void
  183. xfs_finish_ioend(
  184. struct xfs_ioend *ioend)
  185. {
  186. if (atomic_dec_and_test(&ioend->io_remaining)) {
  187. if (ioend->io_type == IO_UNWRITTEN)
  188. queue_work(xfsconvertd_workqueue, &ioend->io_work);
  189. else
  190. queue_work(xfsdatad_workqueue, &ioend->io_work);
  191. }
  192. }
  193. /*
  194. * IO write completion.
  195. */
  196. STATIC void
  197. xfs_end_io(
  198. struct work_struct *work)
  199. {
  200. xfs_ioend_t *ioend = container_of(work, xfs_ioend_t, io_work);
  201. struct xfs_inode *ip = XFS_I(ioend->io_inode);
  202. int error = 0;
  203. /*
  204. * For unwritten extents we need to issue transactions to convert a
  205. * range to normal written extens after the data I/O has finished.
  206. */
  207. if (ioend->io_type == IO_UNWRITTEN &&
  208. likely(!ioend->io_error && !XFS_FORCED_SHUTDOWN(ip->i_mount))) {
  209. error = xfs_iomap_write_unwritten(ip, ioend->io_offset,
  210. ioend->io_size);
  211. if (error)
  212. ioend->io_error = error;
  213. }
  214. /*
  215. * We might have to update the on-disk file size after extending
  216. * writes.
  217. */
  218. if (ioend->io_type != IO_READ) {
  219. error = xfs_setfilesize(ioend);
  220. ASSERT(!error || error == EAGAIN);
  221. }
  222. /*
  223. * If we didn't complete processing of the ioend, requeue it to the
  224. * tail of the workqueue for another attempt later. Otherwise destroy
  225. * it.
  226. */
  227. if (error == EAGAIN) {
  228. atomic_inc(&ioend->io_remaining);
  229. xfs_finish_ioend(ioend);
  230. /* ensure we don't spin on blocked ioends */
  231. delay(1);
  232. } else {
  233. if (ioend->io_iocb)
  234. aio_complete(ioend->io_iocb, ioend->io_result, 0);
  235. xfs_destroy_ioend(ioend);
  236. }
  237. }
  238. /*
  239. * Call IO completion handling in caller context on the final put of an ioend.
  240. */
  241. STATIC void
  242. xfs_finish_ioend_sync(
  243. struct xfs_ioend *ioend)
  244. {
  245. if (atomic_dec_and_test(&ioend->io_remaining))
  246. xfs_end_io(&ioend->io_work);
  247. }
  248. /*
  249. * Allocate and initialise an IO completion structure.
  250. * We need to track unwritten extent write completion here initially.
  251. * We'll need to extend this for updating the ondisk inode size later
  252. * (vs. incore size).
  253. */
  254. STATIC xfs_ioend_t *
  255. xfs_alloc_ioend(
  256. struct inode *inode,
  257. unsigned int type)
  258. {
  259. xfs_ioend_t *ioend;
  260. ioend = mempool_alloc(xfs_ioend_pool, GFP_NOFS);
  261. /*
  262. * Set the count to 1 initially, which will prevent an I/O
  263. * completion callback from happening before we have started
  264. * all the I/O from calling the completion routine too early.
  265. */
  266. atomic_set(&ioend->io_remaining, 1);
  267. ioend->io_error = 0;
  268. ioend->io_list = NULL;
  269. ioend->io_type = type;
  270. ioend->io_inode = inode;
  271. ioend->io_buffer_head = NULL;
  272. ioend->io_buffer_tail = NULL;
  273. atomic_inc(&XFS_I(ioend->io_inode)->i_iocount);
  274. ioend->io_offset = 0;
  275. ioend->io_size = 0;
  276. ioend->io_iocb = NULL;
  277. ioend->io_result = 0;
  278. INIT_WORK(&ioend->io_work, xfs_end_io);
  279. return ioend;
  280. }
  281. STATIC int
  282. xfs_map_blocks(
  283. struct inode *inode,
  284. loff_t offset,
  285. ssize_t count,
  286. struct xfs_bmbt_irec *imap,
  287. int flags)
  288. {
  289. int nmaps = 1;
  290. int new = 0;
  291. return -xfs_iomap(XFS_I(inode), offset, count, flags, imap, &nmaps, &new);
  292. }
  293. STATIC int
  294. xfs_imap_valid(
  295. struct inode *inode,
  296. struct xfs_bmbt_irec *imap,
  297. xfs_off_t offset)
  298. {
  299. offset >>= inode->i_blkbits;
  300. return offset >= imap->br_startoff &&
  301. offset < imap->br_startoff + imap->br_blockcount;
  302. }
  303. /*
  304. * BIO completion handler for buffered IO.
  305. */
  306. STATIC void
  307. xfs_end_bio(
  308. struct bio *bio,
  309. int error)
  310. {
  311. xfs_ioend_t *ioend = bio->bi_private;
  312. ASSERT(atomic_read(&bio->bi_cnt) >= 1);
  313. ioend->io_error = test_bit(BIO_UPTODATE, &bio->bi_flags) ? 0 : error;
  314. /* Toss bio and pass work off to an xfsdatad thread */
  315. bio->bi_private = NULL;
  316. bio->bi_end_io = NULL;
  317. bio_put(bio);
  318. xfs_finish_ioend(ioend);
  319. }
  320. STATIC void
  321. xfs_submit_ioend_bio(
  322. struct writeback_control *wbc,
  323. xfs_ioend_t *ioend,
  324. struct bio *bio)
  325. {
  326. atomic_inc(&ioend->io_remaining);
  327. bio->bi_private = ioend;
  328. bio->bi_end_io = xfs_end_bio;
  329. /*
  330. * If the I/O is beyond EOF we mark the inode dirty immediately
  331. * but don't update the inode size until I/O completion.
  332. */
  333. if (xfs_ioend_new_eof(ioend))
  334. xfs_mark_inode_dirty(XFS_I(ioend->io_inode));
  335. submit_bio(wbc->sync_mode == WB_SYNC_ALL ?
  336. WRITE_SYNC_PLUG : WRITE, bio);
  337. }
  338. STATIC struct bio *
  339. xfs_alloc_ioend_bio(
  340. struct buffer_head *bh)
  341. {
  342. int nvecs = bio_get_nr_vecs(bh->b_bdev);
  343. struct bio *bio = bio_alloc(GFP_NOIO, nvecs);
  344. ASSERT(bio->bi_private == NULL);
  345. bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
  346. bio->bi_bdev = bh->b_bdev;
  347. return bio;
  348. }
  349. STATIC void
  350. xfs_start_buffer_writeback(
  351. struct buffer_head *bh)
  352. {
  353. ASSERT(buffer_mapped(bh));
  354. ASSERT(buffer_locked(bh));
  355. ASSERT(!buffer_delay(bh));
  356. ASSERT(!buffer_unwritten(bh));
  357. mark_buffer_async_write(bh);
  358. set_buffer_uptodate(bh);
  359. clear_buffer_dirty(bh);
  360. }
  361. STATIC void
  362. xfs_start_page_writeback(
  363. struct page *page,
  364. int clear_dirty,
  365. int buffers)
  366. {
  367. ASSERT(PageLocked(page));
  368. ASSERT(!PageWriteback(page));
  369. if (clear_dirty)
  370. clear_page_dirty_for_io(page);
  371. set_page_writeback(page);
  372. unlock_page(page);
  373. /* If no buffers on the page are to be written, finish it here */
  374. if (!buffers)
  375. end_page_writeback(page);
  376. }
  377. static inline int bio_add_buffer(struct bio *bio, struct buffer_head *bh)
  378. {
  379. return bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
  380. }
  381. /*
  382. * Submit all of the bios for all of the ioends we have saved up, covering the
  383. * initial writepage page and also any probed pages.
  384. *
  385. * Because we may have multiple ioends spanning a page, we need to start
  386. * writeback on all the buffers before we submit them for I/O. If we mark the
  387. * buffers as we got, then we can end up with a page that only has buffers
  388. * marked async write and I/O complete on can occur before we mark the other
  389. * buffers async write.
  390. *
  391. * The end result of this is that we trip a bug in end_page_writeback() because
  392. * we call it twice for the one page as the code in end_buffer_async_write()
  393. * assumes that all buffers on the page are started at the same time.
  394. *
  395. * The fix is two passes across the ioend list - one to start writeback on the
  396. * buffer_heads, and then submit them for I/O on the second pass.
  397. */
  398. STATIC void
  399. xfs_submit_ioend(
  400. struct writeback_control *wbc,
  401. xfs_ioend_t *ioend)
  402. {
  403. xfs_ioend_t *head = ioend;
  404. xfs_ioend_t *next;
  405. struct buffer_head *bh;
  406. struct bio *bio;
  407. sector_t lastblock = 0;
  408. /* Pass 1 - start writeback */
  409. do {
  410. next = ioend->io_list;
  411. for (bh = ioend->io_buffer_head; bh; bh = bh->b_private)
  412. xfs_start_buffer_writeback(bh);
  413. } while ((ioend = next) != NULL);
  414. /* Pass 2 - submit I/O */
  415. ioend = head;
  416. do {
  417. next = ioend->io_list;
  418. bio = NULL;
  419. for (bh = ioend->io_buffer_head; bh; bh = bh->b_private) {
  420. if (!bio) {
  421. retry:
  422. bio = xfs_alloc_ioend_bio(bh);
  423. } else if (bh->b_blocknr != lastblock + 1) {
  424. xfs_submit_ioend_bio(wbc, ioend, bio);
  425. goto retry;
  426. }
  427. if (bio_add_buffer(bio, bh) != bh->b_size) {
  428. xfs_submit_ioend_bio(wbc, ioend, bio);
  429. goto retry;
  430. }
  431. lastblock = bh->b_blocknr;
  432. }
  433. if (bio)
  434. xfs_submit_ioend_bio(wbc, ioend, bio);
  435. xfs_finish_ioend(ioend);
  436. } while ((ioend = next) != NULL);
  437. }
  438. /*
  439. * Cancel submission of all buffer_heads so far in this endio.
  440. * Toss the endio too. Only ever called for the initial page
  441. * in a writepage request, so only ever one page.
  442. */
  443. STATIC void
  444. xfs_cancel_ioend(
  445. xfs_ioend_t *ioend)
  446. {
  447. xfs_ioend_t *next;
  448. struct buffer_head *bh, *next_bh;
  449. do {
  450. next = ioend->io_list;
  451. bh = ioend->io_buffer_head;
  452. do {
  453. next_bh = bh->b_private;
  454. clear_buffer_async_write(bh);
  455. unlock_buffer(bh);
  456. } while ((bh = next_bh) != NULL);
  457. xfs_ioend_wake(XFS_I(ioend->io_inode));
  458. mempool_free(ioend, xfs_ioend_pool);
  459. } while ((ioend = next) != NULL);
  460. }
  461. /*
  462. * Test to see if we've been building up a completion structure for
  463. * earlier buffers -- if so, we try to append to this ioend if we
  464. * can, otherwise we finish off any current ioend and start another.
  465. * Return true if we've finished the given ioend.
  466. */
  467. STATIC void
  468. xfs_add_to_ioend(
  469. struct inode *inode,
  470. struct buffer_head *bh,
  471. xfs_off_t offset,
  472. unsigned int type,
  473. xfs_ioend_t **result,
  474. int need_ioend)
  475. {
  476. xfs_ioend_t *ioend = *result;
  477. if (!ioend || need_ioend || type != ioend->io_type) {
  478. xfs_ioend_t *previous = *result;
  479. ioend = xfs_alloc_ioend(inode, type);
  480. ioend->io_offset = offset;
  481. ioend->io_buffer_head = bh;
  482. ioend->io_buffer_tail = bh;
  483. if (previous)
  484. previous->io_list = ioend;
  485. *result = ioend;
  486. } else {
  487. ioend->io_buffer_tail->b_private = bh;
  488. ioend->io_buffer_tail = bh;
  489. }
  490. bh->b_private = NULL;
  491. ioend->io_size += bh->b_size;
  492. }
  493. STATIC void
  494. xfs_map_buffer(
  495. struct inode *inode,
  496. struct buffer_head *bh,
  497. struct xfs_bmbt_irec *imap,
  498. xfs_off_t offset)
  499. {
  500. sector_t bn;
  501. struct xfs_mount *m = XFS_I(inode)->i_mount;
  502. xfs_off_t iomap_offset = XFS_FSB_TO_B(m, imap->br_startoff);
  503. xfs_daddr_t iomap_bn = xfs_fsb_to_db(XFS_I(inode), imap->br_startblock);
  504. ASSERT(imap->br_startblock != HOLESTARTBLOCK);
  505. ASSERT(imap->br_startblock != DELAYSTARTBLOCK);
  506. bn = (iomap_bn >> (inode->i_blkbits - BBSHIFT)) +
  507. ((offset - iomap_offset) >> inode->i_blkbits);
  508. ASSERT(bn || XFS_IS_REALTIME_INODE(XFS_I(inode)));
  509. bh->b_blocknr = bn;
  510. set_buffer_mapped(bh);
  511. }
  512. STATIC void
  513. xfs_map_at_offset(
  514. struct inode *inode,
  515. struct buffer_head *bh,
  516. struct xfs_bmbt_irec *imap,
  517. xfs_off_t offset)
  518. {
  519. ASSERT(imap->br_startblock != HOLESTARTBLOCK);
  520. ASSERT(imap->br_startblock != DELAYSTARTBLOCK);
  521. lock_buffer(bh);
  522. xfs_map_buffer(inode, bh, imap, offset);
  523. bh->b_bdev = xfs_find_bdev_for_inode(inode);
  524. set_buffer_mapped(bh);
  525. clear_buffer_delay(bh);
  526. clear_buffer_unwritten(bh);
  527. }
  528. /*
  529. * Look for a page at index that is suitable for clustering.
  530. */
  531. STATIC unsigned int
  532. xfs_probe_page(
  533. struct page *page,
  534. unsigned int pg_offset)
  535. {
  536. struct buffer_head *bh, *head;
  537. int ret = 0;
  538. if (PageWriteback(page))
  539. return 0;
  540. if (!PageDirty(page))
  541. return 0;
  542. if (!page->mapping)
  543. return 0;
  544. if (!page_has_buffers(page))
  545. return 0;
  546. bh = head = page_buffers(page);
  547. do {
  548. if (!buffer_uptodate(bh))
  549. break;
  550. if (!buffer_mapped(bh))
  551. break;
  552. ret += bh->b_size;
  553. if (ret >= pg_offset)
  554. break;
  555. } while ((bh = bh->b_this_page) != head);
  556. return ret;
  557. }
  558. STATIC size_t
  559. xfs_probe_cluster(
  560. struct inode *inode,
  561. struct page *startpage,
  562. struct buffer_head *bh,
  563. struct buffer_head *head)
  564. {
  565. struct pagevec pvec;
  566. pgoff_t tindex, tlast, tloff;
  567. size_t total = 0;
  568. int done = 0, i;
  569. /* First sum forwards in this page */
  570. do {
  571. if (!buffer_uptodate(bh) || !buffer_mapped(bh))
  572. return total;
  573. total += bh->b_size;
  574. } while ((bh = bh->b_this_page) != head);
  575. /* if we reached the end of the page, sum forwards in following pages */
  576. tlast = i_size_read(inode) >> PAGE_CACHE_SHIFT;
  577. tindex = startpage->index + 1;
  578. /* Prune this back to avoid pathological behavior */
  579. tloff = min(tlast, startpage->index + 64);
  580. pagevec_init(&pvec, 0);
  581. while (!done && tindex <= tloff) {
  582. unsigned len = min_t(pgoff_t, PAGEVEC_SIZE, tlast - tindex + 1);
  583. if (!pagevec_lookup(&pvec, inode->i_mapping, tindex, len))
  584. break;
  585. for (i = 0; i < pagevec_count(&pvec); i++) {
  586. struct page *page = pvec.pages[i];
  587. size_t pg_offset, pg_len = 0;
  588. if (tindex == tlast) {
  589. pg_offset =
  590. i_size_read(inode) & (PAGE_CACHE_SIZE - 1);
  591. if (!pg_offset) {
  592. done = 1;
  593. break;
  594. }
  595. } else
  596. pg_offset = PAGE_CACHE_SIZE;
  597. if (page->index == tindex && trylock_page(page)) {
  598. pg_len = xfs_probe_page(page, pg_offset);
  599. unlock_page(page);
  600. }
  601. if (!pg_len) {
  602. done = 1;
  603. break;
  604. }
  605. total += pg_len;
  606. tindex++;
  607. }
  608. pagevec_release(&pvec);
  609. cond_resched();
  610. }
  611. return total;
  612. }
  613. /*
  614. * Test if a given page is suitable for writing as part of an unwritten
  615. * or delayed allocate extent.
  616. */
  617. STATIC int
  618. xfs_is_delayed_page(
  619. struct page *page,
  620. unsigned int type)
  621. {
  622. if (PageWriteback(page))
  623. return 0;
  624. if (page->mapping && page_has_buffers(page)) {
  625. struct buffer_head *bh, *head;
  626. int acceptable = 0;
  627. bh = head = page_buffers(page);
  628. do {
  629. if (buffer_unwritten(bh))
  630. acceptable = (type == IO_UNWRITTEN);
  631. else if (buffer_delay(bh))
  632. acceptable = (type == IO_DELAY);
  633. else if (buffer_dirty(bh) && buffer_mapped(bh))
  634. acceptable = (type == IO_NEW);
  635. else
  636. break;
  637. } while ((bh = bh->b_this_page) != head);
  638. if (acceptable)
  639. return 1;
  640. }
  641. return 0;
  642. }
  643. /*
  644. * Allocate & map buffers for page given the extent map. Write it out.
  645. * except for the original page of a writepage, this is called on
  646. * delalloc/unwritten pages only, for the original page it is possible
  647. * that the page has no mapping at all.
  648. */
  649. STATIC int
  650. xfs_convert_page(
  651. struct inode *inode,
  652. struct page *page,
  653. loff_t tindex,
  654. struct xfs_bmbt_irec *imap,
  655. xfs_ioend_t **ioendp,
  656. struct writeback_control *wbc,
  657. int all_bh)
  658. {
  659. struct buffer_head *bh, *head;
  660. xfs_off_t end_offset;
  661. unsigned long p_offset;
  662. unsigned int type;
  663. int len, page_dirty;
  664. int count = 0, done = 0, uptodate = 1;
  665. xfs_off_t offset = page_offset(page);
  666. if (page->index != tindex)
  667. goto fail;
  668. if (!trylock_page(page))
  669. goto fail;
  670. if (PageWriteback(page))
  671. goto fail_unlock_page;
  672. if (page->mapping != inode->i_mapping)
  673. goto fail_unlock_page;
  674. if (!xfs_is_delayed_page(page, (*ioendp)->io_type))
  675. goto fail_unlock_page;
  676. /*
  677. * page_dirty is initially a count of buffers on the page before
  678. * EOF and is decremented as we move each into a cleanable state.
  679. *
  680. * Derivation:
  681. *
  682. * End offset is the highest offset that this page should represent.
  683. * If we are on the last page, (end_offset & (PAGE_CACHE_SIZE - 1))
  684. * will evaluate non-zero and be less than PAGE_CACHE_SIZE and
  685. * hence give us the correct page_dirty count. On any other page,
  686. * it will be zero and in that case we need page_dirty to be the
  687. * count of buffers on the page.
  688. */
  689. end_offset = min_t(unsigned long long,
  690. (xfs_off_t)(page->index + 1) << PAGE_CACHE_SHIFT,
  691. i_size_read(inode));
  692. len = 1 << inode->i_blkbits;
  693. p_offset = min_t(unsigned long, end_offset & (PAGE_CACHE_SIZE - 1),
  694. PAGE_CACHE_SIZE);
  695. p_offset = p_offset ? roundup(p_offset, len) : PAGE_CACHE_SIZE;
  696. page_dirty = p_offset / len;
  697. bh = head = page_buffers(page);
  698. do {
  699. if (offset >= end_offset)
  700. break;
  701. if (!buffer_uptodate(bh))
  702. uptodate = 0;
  703. if (!(PageUptodate(page) || buffer_uptodate(bh))) {
  704. done = 1;
  705. continue;
  706. }
  707. if (buffer_unwritten(bh) || buffer_delay(bh)) {
  708. if (buffer_unwritten(bh))
  709. type = IO_UNWRITTEN;
  710. else
  711. type = IO_DELAY;
  712. if (!xfs_imap_valid(inode, imap, offset)) {
  713. done = 1;
  714. continue;
  715. }
  716. ASSERT(imap->br_startblock != HOLESTARTBLOCK);
  717. ASSERT(imap->br_startblock != DELAYSTARTBLOCK);
  718. xfs_map_at_offset(inode, bh, imap, offset);
  719. xfs_add_to_ioend(inode, bh, offset, type,
  720. ioendp, done);
  721. page_dirty--;
  722. count++;
  723. } else {
  724. type = IO_NEW;
  725. if (buffer_mapped(bh) && all_bh) {
  726. lock_buffer(bh);
  727. xfs_add_to_ioend(inode, bh, offset,
  728. type, ioendp, done);
  729. count++;
  730. page_dirty--;
  731. } else {
  732. done = 1;
  733. }
  734. }
  735. } while (offset += len, (bh = bh->b_this_page) != head);
  736. if (uptodate && bh == head)
  737. SetPageUptodate(page);
  738. if (count) {
  739. if (--wbc->nr_to_write <= 0 &&
  740. wbc->sync_mode == WB_SYNC_NONE)
  741. done = 1;
  742. }
  743. xfs_start_page_writeback(page, !page_dirty, count);
  744. return done;
  745. fail_unlock_page:
  746. unlock_page(page);
  747. fail:
  748. return 1;
  749. }
  750. /*
  751. * Convert & write out a cluster of pages in the same extent as defined
  752. * by mp and following the start page.
  753. */
  754. STATIC void
  755. xfs_cluster_write(
  756. struct inode *inode,
  757. pgoff_t tindex,
  758. struct xfs_bmbt_irec *imap,
  759. xfs_ioend_t **ioendp,
  760. struct writeback_control *wbc,
  761. int all_bh,
  762. pgoff_t tlast)
  763. {
  764. struct pagevec pvec;
  765. int done = 0, i;
  766. pagevec_init(&pvec, 0);
  767. while (!done && tindex <= tlast) {
  768. unsigned len = min_t(pgoff_t, PAGEVEC_SIZE, tlast - tindex + 1);
  769. if (!pagevec_lookup(&pvec, inode->i_mapping, tindex, len))
  770. break;
  771. for (i = 0; i < pagevec_count(&pvec); i++) {
  772. done = xfs_convert_page(inode, pvec.pages[i], tindex++,
  773. imap, ioendp, wbc, all_bh);
  774. if (done)
  775. break;
  776. }
  777. pagevec_release(&pvec);
  778. cond_resched();
  779. }
  780. }
  781. STATIC void
  782. xfs_vm_invalidatepage(
  783. struct page *page,
  784. unsigned long offset)
  785. {
  786. trace_xfs_invalidatepage(page->mapping->host, page, offset);
  787. block_invalidatepage(page, offset);
  788. }
  789. /*
  790. * If the page has delalloc buffers on it, we need to punch them out before we
  791. * invalidate the page. If we don't, we leave a stale delalloc mapping on the
  792. * inode that can trip a BUG() in xfs_get_blocks() later on if a direct IO read
  793. * is done on that same region - the delalloc extent is returned when none is
  794. * supposed to be there.
  795. *
  796. * We prevent this by truncating away the delalloc regions on the page before
  797. * invalidating it. Because they are delalloc, we can do this without needing a
  798. * transaction. Indeed - if we get ENOSPC errors, we have to be able to do this
  799. * truncation without a transaction as there is no space left for block
  800. * reservation (typically why we see a ENOSPC in writeback).
  801. *
  802. * This is not a performance critical path, so for now just do the punching a
  803. * buffer head at a time.
  804. */
  805. STATIC void
  806. xfs_aops_discard_page(
  807. struct page *page)
  808. {
  809. struct inode *inode = page->mapping->host;
  810. struct xfs_inode *ip = XFS_I(inode);
  811. struct buffer_head *bh, *head;
  812. loff_t offset = page_offset(page);
  813. if (!xfs_is_delayed_page(page, IO_DELAY))
  814. goto out_invalidate;
  815. if (XFS_FORCED_SHUTDOWN(ip->i_mount))
  816. goto out_invalidate;
  817. xfs_fs_cmn_err(CE_ALERT, ip->i_mount,
  818. "page discard on page %p, inode 0x%llx, offset %llu.",
  819. page, ip->i_ino, offset);
  820. xfs_ilock(ip, XFS_ILOCK_EXCL);
  821. bh = head = page_buffers(page);
  822. do {
  823. int error;
  824. xfs_fileoff_t start_fsb;
  825. if (!buffer_delay(bh))
  826. goto next_buffer;
  827. start_fsb = XFS_B_TO_FSBT(ip->i_mount, offset);
  828. error = xfs_bmap_punch_delalloc_range(ip, start_fsb, 1);
  829. if (error) {
  830. /* something screwed, just bail */
  831. if (!XFS_FORCED_SHUTDOWN(ip->i_mount)) {
  832. xfs_fs_cmn_err(CE_ALERT, ip->i_mount,
  833. "page discard unable to remove delalloc mapping.");
  834. }
  835. break;
  836. }
  837. next_buffer:
  838. offset += 1 << inode->i_blkbits;
  839. } while ((bh = bh->b_this_page) != head);
  840. xfs_iunlock(ip, XFS_ILOCK_EXCL);
  841. out_invalidate:
  842. xfs_vm_invalidatepage(page, 0);
  843. return;
  844. }
  845. /*
  846. * Write out a dirty page.
  847. *
  848. * For delalloc space on the page we need to allocate space and flush it.
  849. * For unwritten space on the page we need to start the conversion to
  850. * regular allocated space.
  851. * For any other dirty buffer heads on the page we should flush them.
  852. *
  853. * If we detect that a transaction would be required to flush the page, we
  854. * have to check the process flags first, if we are already in a transaction
  855. * or disk I/O during allocations is off, we need to fail the writepage and
  856. * redirty the page.
  857. */
  858. STATIC int
  859. xfs_vm_writepage(
  860. struct page *page,
  861. struct writeback_control *wbc)
  862. {
  863. struct inode *inode = page->mapping->host;
  864. int delalloc, unwritten;
  865. struct buffer_head *bh, *head;
  866. struct xfs_bmbt_irec imap;
  867. xfs_ioend_t *ioend = NULL, *iohead = NULL;
  868. loff_t offset;
  869. unsigned int type;
  870. __uint64_t end_offset;
  871. pgoff_t end_index, last_index;
  872. ssize_t size, len;
  873. int flags, err, imap_valid = 0, uptodate = 1;
  874. int count = 0;
  875. int all_bh = 0;
  876. trace_xfs_writepage(inode, page, 0);
  877. ASSERT(page_has_buffers(page));
  878. /*
  879. * Refuse to write the page out if we are called from reclaim context.
  880. *
  881. * This avoids stack overflows when called from deeply used stacks in
  882. * random callers for direct reclaim or memcg reclaim. We explicitly
  883. * allow reclaim from kswapd as the stack usage there is relatively low.
  884. *
  885. * This should really be done by the core VM, but until that happens
  886. * filesystems like XFS, btrfs and ext4 have to take care of this
  887. * by themselves.
  888. */
  889. if ((current->flags & (PF_MEMALLOC|PF_KSWAPD)) == PF_MEMALLOC)
  890. goto redirty;
  891. /*
  892. * We need a transaction if there are delalloc or unwritten buffers
  893. * on the page.
  894. *
  895. * If we need a transaction and the process flags say we are already
  896. * in a transaction, or no IO is allowed then mark the page dirty
  897. * again and leave the page as is.
  898. */
  899. xfs_count_page_state(page, &delalloc, &unwritten);
  900. if ((current->flags & PF_FSTRANS) && (delalloc || unwritten))
  901. goto redirty;
  902. /* Is this page beyond the end of the file? */
  903. offset = i_size_read(inode);
  904. end_index = offset >> PAGE_CACHE_SHIFT;
  905. last_index = (offset - 1) >> PAGE_CACHE_SHIFT;
  906. if (page->index >= end_index) {
  907. if ((page->index >= end_index + 1) ||
  908. !(i_size_read(inode) & (PAGE_CACHE_SIZE - 1))) {
  909. unlock_page(page);
  910. return 0;
  911. }
  912. }
  913. end_offset = min_t(unsigned long long,
  914. (xfs_off_t)(page->index + 1) << PAGE_CACHE_SHIFT,
  915. offset);
  916. len = 1 << inode->i_blkbits;
  917. bh = head = page_buffers(page);
  918. offset = page_offset(page);
  919. flags = BMAPI_READ;
  920. type = IO_NEW;
  921. do {
  922. if (offset >= end_offset)
  923. break;
  924. if (!buffer_uptodate(bh))
  925. uptodate = 0;
  926. /*
  927. * set_page_dirty dirties all buffers in a page, independent
  928. * of their state. The dirty state however is entirely
  929. * meaningless for holes (!mapped && uptodate), so skip
  930. * buffers covering holes here.
  931. */
  932. if (!buffer_mapped(bh) && buffer_uptodate(bh)) {
  933. imap_valid = 0;
  934. continue;
  935. }
  936. if (imap_valid)
  937. imap_valid = xfs_imap_valid(inode, &imap, offset);
  938. if (buffer_unwritten(bh) || buffer_delay(bh)) {
  939. int new_ioend = 0;
  940. if (buffer_unwritten(bh)) {
  941. if (type != IO_UNWRITTEN) {
  942. type = IO_UNWRITTEN;
  943. imap_valid = 0;
  944. }
  945. flags = BMAPI_WRITE | BMAPI_IGNSTATE;
  946. } else if (buffer_delay(bh)) {
  947. if (type != IO_DELAY) {
  948. type = IO_DELAY;
  949. imap_valid = 0;
  950. }
  951. flags = BMAPI_ALLOCATE;
  952. if (wbc->sync_mode == WB_SYNC_NONE)
  953. flags |= BMAPI_TRYLOCK;
  954. }
  955. if (!imap_valid) {
  956. /*
  957. * If we didn't have a valid mapping then we
  958. * need to ensure that we put the new mapping
  959. * in a new ioend structure. This needs to be
  960. * done to ensure that the ioends correctly
  961. * reflect the block mappings at io completion
  962. * for unwritten extent conversion.
  963. */
  964. new_ioend = 1;
  965. err = xfs_map_blocks(inode, offset, len,
  966. &imap, flags);
  967. if (err)
  968. goto error;
  969. imap_valid = xfs_imap_valid(inode, &imap,
  970. offset);
  971. }
  972. if (imap_valid) {
  973. xfs_map_at_offset(inode, bh, &imap, offset);
  974. xfs_add_to_ioend(inode, bh, offset, type,
  975. &ioend, new_ioend);
  976. count++;
  977. }
  978. } else if (buffer_uptodate(bh)) {
  979. /*
  980. * we got here because the buffer is already mapped.
  981. * That means it must already have extents allocated
  982. * underneath it. Map the extent by reading it.
  983. */
  984. if (flags != BMAPI_READ) {
  985. flags = BMAPI_READ;
  986. imap_valid = 0;
  987. }
  988. if (!imap_valid) {
  989. size = xfs_probe_cluster(inode, page, bh, head);
  990. err = xfs_map_blocks(inode, offset, size,
  991. &imap, flags);
  992. if (err)
  993. goto error;
  994. imap_valid = xfs_imap_valid(inode, &imap,
  995. offset);
  996. }
  997. /*
  998. * We set the type to IO_NEW in case we are doing a
  999. * small write at EOF that is extending the file but
  1000. * without needing an allocation. We need to update the
  1001. * file size on I/O completion in this case so it is
  1002. * the same case as having just allocated a new extent
  1003. * that we are writing into for the first time.
  1004. */
  1005. type = IO_NEW;
  1006. if (trylock_buffer(bh)) {
  1007. if (imap_valid)
  1008. all_bh = 1;
  1009. xfs_add_to_ioend(inode, bh, offset, type,
  1010. &ioend, !imap_valid);
  1011. count++;
  1012. } else {
  1013. imap_valid = 0;
  1014. }
  1015. } else if (PageUptodate(page)) {
  1016. ASSERT(buffer_mapped(bh));
  1017. imap_valid = 0;
  1018. }
  1019. if (!iohead)
  1020. iohead = ioend;
  1021. } while (offset += len, ((bh = bh->b_this_page) != head));
  1022. if (uptodate && bh == head)
  1023. SetPageUptodate(page);
  1024. xfs_start_page_writeback(page, 1, count);
  1025. if (ioend && imap_valid) {
  1026. xfs_off_t end_index;
  1027. end_index = imap.br_startoff + imap.br_blockcount;
  1028. /* to bytes */
  1029. end_index <<= inode->i_blkbits;
  1030. /* to pages */
  1031. end_index = (end_index - 1) >> PAGE_CACHE_SHIFT;
  1032. /* check against file size */
  1033. if (end_index > last_index)
  1034. end_index = last_index;
  1035. xfs_cluster_write(inode, page->index + 1, &imap, &ioend,
  1036. wbc, all_bh, end_index);
  1037. }
  1038. if (iohead)
  1039. xfs_submit_ioend(wbc, iohead);
  1040. return 0;
  1041. error:
  1042. if (iohead)
  1043. xfs_cancel_ioend(iohead);
  1044. if (err == -EAGAIN)
  1045. goto redirty;
  1046. xfs_aops_discard_page(page);
  1047. ClearPageUptodate(page);
  1048. unlock_page(page);
  1049. return err;
  1050. redirty:
  1051. redirty_page_for_writepage(wbc, page);
  1052. unlock_page(page);
  1053. return 0;
  1054. }
  1055. STATIC int
  1056. xfs_vm_writepages(
  1057. struct address_space *mapping,
  1058. struct writeback_control *wbc)
  1059. {
  1060. xfs_iflags_clear(XFS_I(mapping->host), XFS_ITRUNCATED);
  1061. return generic_writepages(mapping, wbc);
  1062. }
  1063. /*
  1064. * Called to move a page into cleanable state - and from there
  1065. * to be released. The page should already be clean. We always
  1066. * have buffer heads in this call.
  1067. *
  1068. * Returns 1 if the page is ok to release, 0 otherwise.
  1069. */
  1070. STATIC int
  1071. xfs_vm_releasepage(
  1072. struct page *page,
  1073. gfp_t gfp_mask)
  1074. {
  1075. int delalloc, unwritten;
  1076. trace_xfs_releasepage(page->mapping->host, page, 0);
  1077. xfs_count_page_state(page, &delalloc, &unwritten);
  1078. if (WARN_ON(delalloc))
  1079. return 0;
  1080. if (WARN_ON(unwritten))
  1081. return 0;
  1082. return try_to_free_buffers(page);
  1083. }
  1084. STATIC int
  1085. __xfs_get_blocks(
  1086. struct inode *inode,
  1087. sector_t iblock,
  1088. struct buffer_head *bh_result,
  1089. int create,
  1090. int direct)
  1091. {
  1092. int flags = create ? BMAPI_WRITE : BMAPI_READ;
  1093. struct xfs_bmbt_irec imap;
  1094. xfs_off_t offset;
  1095. ssize_t size;
  1096. int nimap = 1;
  1097. int new = 0;
  1098. int error;
  1099. offset = (xfs_off_t)iblock << inode->i_blkbits;
  1100. ASSERT(bh_result->b_size >= (1 << inode->i_blkbits));
  1101. size = bh_result->b_size;
  1102. if (!create && direct && offset >= i_size_read(inode))
  1103. return 0;
  1104. if (direct && create)
  1105. flags |= BMAPI_DIRECT;
  1106. error = xfs_iomap(XFS_I(inode), offset, size, flags, &imap, &nimap,
  1107. &new);
  1108. if (error)
  1109. return -error;
  1110. if (nimap == 0)
  1111. return 0;
  1112. if (imap.br_startblock != HOLESTARTBLOCK &&
  1113. imap.br_startblock != DELAYSTARTBLOCK) {
  1114. /*
  1115. * For unwritten extents do not report a disk address on
  1116. * the read case (treat as if we're reading into a hole).
  1117. */
  1118. if (create || !ISUNWRITTEN(&imap))
  1119. xfs_map_buffer(inode, bh_result, &imap, offset);
  1120. if (create && ISUNWRITTEN(&imap)) {
  1121. if (direct)
  1122. bh_result->b_private = inode;
  1123. set_buffer_unwritten(bh_result);
  1124. }
  1125. }
  1126. /*
  1127. * If this is a realtime file, data may be on a different device.
  1128. * to that pointed to from the buffer_head b_bdev currently.
  1129. */
  1130. bh_result->b_bdev = xfs_find_bdev_for_inode(inode);
  1131. /*
  1132. * If we previously allocated a block out beyond eof and we are now
  1133. * coming back to use it then we will need to flag it as new even if it
  1134. * has a disk address.
  1135. *
  1136. * With sub-block writes into unwritten extents we also need to mark
  1137. * the buffer as new so that the unwritten parts of the buffer gets
  1138. * correctly zeroed.
  1139. */
  1140. if (create &&
  1141. ((!buffer_mapped(bh_result) && !buffer_uptodate(bh_result)) ||
  1142. (offset >= i_size_read(inode)) ||
  1143. (new || ISUNWRITTEN(&imap))))
  1144. set_buffer_new(bh_result);
  1145. if (imap.br_startblock == DELAYSTARTBLOCK) {
  1146. BUG_ON(direct);
  1147. if (create) {
  1148. set_buffer_uptodate(bh_result);
  1149. set_buffer_mapped(bh_result);
  1150. set_buffer_delay(bh_result);
  1151. }
  1152. }
  1153. /*
  1154. * If this is O_DIRECT or the mpage code calling tell them how large
  1155. * the mapping is, so that we can avoid repeated get_blocks calls.
  1156. */
  1157. if (direct || size > (1 << inode->i_blkbits)) {
  1158. xfs_off_t mapping_size;
  1159. mapping_size = imap.br_startoff + imap.br_blockcount - iblock;
  1160. mapping_size <<= inode->i_blkbits;
  1161. ASSERT(mapping_size > 0);
  1162. if (mapping_size > size)
  1163. mapping_size = size;
  1164. if (mapping_size > LONG_MAX)
  1165. mapping_size = LONG_MAX;
  1166. bh_result->b_size = mapping_size;
  1167. }
  1168. return 0;
  1169. }
  1170. int
  1171. xfs_get_blocks(
  1172. struct inode *inode,
  1173. sector_t iblock,
  1174. struct buffer_head *bh_result,
  1175. int create)
  1176. {
  1177. return __xfs_get_blocks(inode, iblock, bh_result, create, 0);
  1178. }
  1179. STATIC int
  1180. xfs_get_blocks_direct(
  1181. struct inode *inode,
  1182. sector_t iblock,
  1183. struct buffer_head *bh_result,
  1184. int create)
  1185. {
  1186. return __xfs_get_blocks(inode, iblock, bh_result, create, 1);
  1187. }
  1188. /*
  1189. * Complete a direct I/O write request.
  1190. *
  1191. * If the private argument is non-NULL __xfs_get_blocks signals us that we
  1192. * need to issue a transaction to convert the range from unwritten to written
  1193. * extents. In case this is regular synchronous I/O we just call xfs_end_io
  1194. * to do this and we are done. But in case this was a successfull AIO
  1195. * request this handler is called from interrupt context, from which we
  1196. * can't start transactions. In that case offload the I/O completion to
  1197. * the workqueues we also use for buffered I/O completion.
  1198. */
  1199. STATIC void
  1200. xfs_end_io_direct_write(
  1201. struct kiocb *iocb,
  1202. loff_t offset,
  1203. ssize_t size,
  1204. void *private,
  1205. int ret,
  1206. bool is_async)
  1207. {
  1208. struct xfs_ioend *ioend = iocb->private;
  1209. /*
  1210. * blockdev_direct_IO can return an error even after the I/O
  1211. * completion handler was called. Thus we need to protect
  1212. * against double-freeing.
  1213. */
  1214. iocb->private = NULL;
  1215. ioend->io_offset = offset;
  1216. ioend->io_size = size;
  1217. if (private && size > 0)
  1218. ioend->io_type = IO_UNWRITTEN;
  1219. if (is_async) {
  1220. /*
  1221. * If we are converting an unwritten extent we need to delay
  1222. * the AIO completion until after the unwrittent extent
  1223. * conversion has completed, otherwise do it ASAP.
  1224. */
  1225. if (ioend->io_type == IO_UNWRITTEN) {
  1226. ioend->io_iocb = iocb;
  1227. ioend->io_result = ret;
  1228. } else {
  1229. aio_complete(iocb, ret, 0);
  1230. }
  1231. xfs_finish_ioend(ioend);
  1232. } else {
  1233. xfs_finish_ioend_sync(ioend);
  1234. }
  1235. }
  1236. STATIC ssize_t
  1237. xfs_vm_direct_IO(
  1238. int rw,
  1239. struct kiocb *iocb,
  1240. const struct iovec *iov,
  1241. loff_t offset,
  1242. unsigned long nr_segs)
  1243. {
  1244. struct inode *inode = iocb->ki_filp->f_mapping->host;
  1245. struct block_device *bdev = xfs_find_bdev_for_inode(inode);
  1246. ssize_t ret;
  1247. if (rw & WRITE) {
  1248. iocb->private = xfs_alloc_ioend(inode, IO_NEW);
  1249. ret = __blockdev_direct_IO(rw, iocb, inode, bdev, iov,
  1250. offset, nr_segs,
  1251. xfs_get_blocks_direct,
  1252. xfs_end_io_direct_write, NULL, 0);
  1253. if (ret != -EIOCBQUEUED && iocb->private)
  1254. xfs_destroy_ioend(iocb->private);
  1255. } else {
  1256. ret = __blockdev_direct_IO(rw, iocb, inode, bdev, iov,
  1257. offset, nr_segs,
  1258. xfs_get_blocks_direct,
  1259. NULL, NULL, 0);
  1260. }
  1261. return ret;
  1262. }
  1263. STATIC void
  1264. xfs_vm_write_failed(
  1265. struct address_space *mapping,
  1266. loff_t to)
  1267. {
  1268. struct inode *inode = mapping->host;
  1269. if (to > inode->i_size) {
  1270. /*
  1271. * punch out the delalloc blocks we have already allocated. We
  1272. * don't call xfs_setattr() to do this as we may be in the
  1273. * middle of a multi-iovec write and so the vfs inode->i_size
  1274. * will not match the xfs ip->i_size and so it will zero too
  1275. * much. Hence we jus truncate the page cache to zero what is
  1276. * necessary and punch the delalloc blocks directly.
  1277. */
  1278. struct xfs_inode *ip = XFS_I(inode);
  1279. xfs_fileoff_t start_fsb;
  1280. xfs_fileoff_t end_fsb;
  1281. int error;
  1282. truncate_pagecache(inode, to, inode->i_size);
  1283. /*
  1284. * Check if there are any blocks that are outside of i_size
  1285. * that need to be trimmed back.
  1286. */
  1287. start_fsb = XFS_B_TO_FSB(ip->i_mount, inode->i_size) + 1;
  1288. end_fsb = XFS_B_TO_FSB(ip->i_mount, to);
  1289. if (end_fsb <= start_fsb)
  1290. return;
  1291. xfs_ilock(ip, XFS_ILOCK_EXCL);
  1292. error = xfs_bmap_punch_delalloc_range(ip, start_fsb,
  1293. end_fsb - start_fsb);
  1294. if (error) {
  1295. /* something screwed, just bail */
  1296. if (!XFS_FORCED_SHUTDOWN(ip->i_mount)) {
  1297. xfs_fs_cmn_err(CE_ALERT, ip->i_mount,
  1298. "xfs_vm_write_failed: unable to clean up ino %lld",
  1299. ip->i_ino);
  1300. }
  1301. }
  1302. xfs_iunlock(ip, XFS_ILOCK_EXCL);
  1303. }
  1304. }
  1305. STATIC int
  1306. xfs_vm_write_begin(
  1307. struct file *file,
  1308. struct address_space *mapping,
  1309. loff_t pos,
  1310. unsigned len,
  1311. unsigned flags,
  1312. struct page **pagep,
  1313. void **fsdata)
  1314. {
  1315. int ret;
  1316. ret = block_write_begin(mapping, pos, len, flags | AOP_FLAG_NOFS,
  1317. pagep, xfs_get_blocks);
  1318. if (unlikely(ret))
  1319. xfs_vm_write_failed(mapping, pos + len);
  1320. return ret;
  1321. }
  1322. STATIC int
  1323. xfs_vm_write_end(
  1324. struct file *file,
  1325. struct address_space *mapping,
  1326. loff_t pos,
  1327. unsigned len,
  1328. unsigned copied,
  1329. struct page *page,
  1330. void *fsdata)
  1331. {
  1332. int ret;
  1333. ret = generic_write_end(file, mapping, pos, len, copied, page, fsdata);
  1334. if (unlikely(ret < len))
  1335. xfs_vm_write_failed(mapping, pos + len);
  1336. return ret;
  1337. }
  1338. STATIC sector_t
  1339. xfs_vm_bmap(
  1340. struct address_space *mapping,
  1341. sector_t block)
  1342. {
  1343. struct inode *inode = (struct inode *)mapping->host;
  1344. struct xfs_inode *ip = XFS_I(inode);
  1345. trace_xfs_vm_bmap(XFS_I(inode));
  1346. xfs_ilock(ip, XFS_IOLOCK_SHARED);
  1347. xfs_flush_pages(ip, (xfs_off_t)0, -1, 0, FI_REMAPF);
  1348. xfs_iunlock(ip, XFS_IOLOCK_SHARED);
  1349. return generic_block_bmap(mapping, block, xfs_get_blocks);
  1350. }
  1351. STATIC int
  1352. xfs_vm_readpage(
  1353. struct file *unused,
  1354. struct page *page)
  1355. {
  1356. return mpage_readpage(page, xfs_get_blocks);
  1357. }
  1358. STATIC int
  1359. xfs_vm_readpages(
  1360. struct file *unused,
  1361. struct address_space *mapping,
  1362. struct list_head *pages,
  1363. unsigned nr_pages)
  1364. {
  1365. return mpage_readpages(mapping, pages, nr_pages, xfs_get_blocks);
  1366. }
  1367. const struct address_space_operations xfs_address_space_operations = {
  1368. .readpage = xfs_vm_readpage,
  1369. .readpages = xfs_vm_readpages,
  1370. .writepage = xfs_vm_writepage,
  1371. .writepages = xfs_vm_writepages,
  1372. .sync_page = block_sync_page,
  1373. .releasepage = xfs_vm_releasepage,
  1374. .invalidatepage = xfs_vm_invalidatepage,
  1375. .write_begin = xfs_vm_write_begin,
  1376. .write_end = xfs_vm_write_end,
  1377. .bmap = xfs_vm_bmap,
  1378. .direct_IO = xfs_vm_direct_IO,
  1379. .migratepage = buffer_migrate_page,
  1380. .is_partially_uptodate = block_is_partially_uptodate,
  1381. .error_remove_page = generic_error_remove_page,
  1382. };