xfs_aops.c 37 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515
  1. /*
  2. * Copyright (c) 2000-2005 Silicon Graphics, Inc.
  3. * All Rights Reserved.
  4. *
  5. * This program is free software; you can redistribute it and/or
  6. * modify it under the terms of the GNU General Public License as
  7. * published by the Free Software Foundation.
  8. *
  9. * This program is distributed in the hope that it would be useful,
  10. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. * GNU General Public License for more details.
  13. *
  14. * You should have received a copy of the GNU General Public License
  15. * along with this program; if not, write the Free Software Foundation,
  16. * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  17. */
  18. #include "xfs.h"
  19. #include "xfs_bit.h"
  20. #include "xfs_log.h"
  21. #include "xfs_inum.h"
  22. #include "xfs_sb.h"
  23. #include "xfs_ag.h"
  24. #include "xfs_trans.h"
  25. #include "xfs_mount.h"
  26. #include "xfs_bmap_btree.h"
  27. #include "xfs_dinode.h"
  28. #include "xfs_inode.h"
  29. #include "xfs_alloc.h"
  30. #include "xfs_error.h"
  31. #include "xfs_rw.h"
  32. #include "xfs_iomap.h"
  33. #include "xfs_vnodeops.h"
  34. #include "xfs_trace.h"
  35. #include "xfs_bmap.h"
  36. #include <linux/gfp.h>
  37. #include <linux/mpage.h>
  38. #include <linux/pagevec.h>
  39. #include <linux/writeback.h>
  40. /*
  41. * Prime number of hash buckets since address is used as the key.
  42. */
  43. #define NVSYNC 37
  44. #define to_ioend_wq(v) (&xfs_ioend_wq[((unsigned long)v) % NVSYNC])
  45. static wait_queue_head_t xfs_ioend_wq[NVSYNC];
  46. void __init
  47. xfs_ioend_init(void)
  48. {
  49. int i;
  50. for (i = 0; i < NVSYNC; i++)
  51. init_waitqueue_head(&xfs_ioend_wq[i]);
  52. }
  53. void
  54. xfs_ioend_wait(
  55. xfs_inode_t *ip)
  56. {
  57. wait_queue_head_t *wq = to_ioend_wq(ip);
  58. wait_event(*wq, (atomic_read(&ip->i_iocount) == 0));
  59. }
  60. STATIC void
  61. xfs_ioend_wake(
  62. xfs_inode_t *ip)
  63. {
  64. if (atomic_dec_and_test(&ip->i_iocount))
  65. wake_up(to_ioend_wq(ip));
  66. }
  67. void
  68. xfs_count_page_state(
  69. struct page *page,
  70. int *delalloc,
  71. int *unwritten)
  72. {
  73. struct buffer_head *bh, *head;
  74. *delalloc = *unwritten = 0;
  75. bh = head = page_buffers(page);
  76. do {
  77. if (buffer_unwritten(bh))
  78. (*unwritten) = 1;
  79. else if (buffer_delay(bh))
  80. (*delalloc) = 1;
  81. } while ((bh = bh->b_this_page) != head);
  82. }
  83. STATIC struct block_device *
  84. xfs_find_bdev_for_inode(
  85. struct inode *inode)
  86. {
  87. struct xfs_inode *ip = XFS_I(inode);
  88. struct xfs_mount *mp = ip->i_mount;
  89. if (XFS_IS_REALTIME_INODE(ip))
  90. return mp->m_rtdev_targp->bt_bdev;
  91. else
  92. return mp->m_ddev_targp->bt_bdev;
  93. }
  94. /*
  95. * We're now finished for good with this ioend structure.
  96. * Update the page state via the associated buffer_heads,
  97. * release holds on the inode and bio, and finally free
  98. * up memory. Do not use the ioend after this.
  99. */
  100. STATIC void
  101. xfs_destroy_ioend(
  102. xfs_ioend_t *ioend)
  103. {
  104. struct buffer_head *bh, *next;
  105. struct xfs_inode *ip = XFS_I(ioend->io_inode);
  106. for (bh = ioend->io_buffer_head; bh; bh = next) {
  107. next = bh->b_private;
  108. bh->b_end_io(bh, !ioend->io_error);
  109. }
  110. /*
  111. * Volume managers supporting multiple paths can send back ENODEV
  112. * when the final path disappears. In this case continuing to fill
  113. * the page cache with dirty data which cannot be written out is
  114. * evil, so prevent that.
  115. */
  116. if (unlikely(ioend->io_error == -ENODEV)) {
  117. xfs_do_force_shutdown(ip->i_mount, SHUTDOWN_DEVICE_REQ,
  118. __FILE__, __LINE__);
  119. }
  120. xfs_ioend_wake(ip);
  121. mempool_free(ioend, xfs_ioend_pool);
  122. }
  123. /*
  124. * If the end of the current ioend is beyond the current EOF,
  125. * return the new EOF value, otherwise zero.
  126. */
  127. STATIC xfs_fsize_t
  128. xfs_ioend_new_eof(
  129. xfs_ioend_t *ioend)
  130. {
  131. xfs_inode_t *ip = XFS_I(ioend->io_inode);
  132. xfs_fsize_t isize;
  133. xfs_fsize_t bsize;
  134. bsize = ioend->io_offset + ioend->io_size;
  135. isize = MAX(ip->i_size, ip->i_new_size);
  136. isize = MIN(isize, bsize);
  137. return isize > ip->i_d.di_size ? isize : 0;
  138. }
  139. /*
  140. * Update on-disk file size now that data has been written to disk. The
  141. * current in-memory file size is i_size. If a write is beyond eof i_new_size
  142. * will be the intended file size until i_size is updated. If this write does
  143. * not extend all the way to the valid file size then restrict this update to
  144. * the end of the write.
  145. *
  146. * This function does not block as blocking on the inode lock in IO completion
  147. * can lead to IO completion order dependency deadlocks.. If it can't get the
  148. * inode ilock it will return EAGAIN. Callers must handle this.
  149. */
  150. STATIC int
  151. xfs_setfilesize(
  152. xfs_ioend_t *ioend)
  153. {
  154. xfs_inode_t *ip = XFS_I(ioend->io_inode);
  155. xfs_fsize_t isize;
  156. if (unlikely(ioend->io_error))
  157. return 0;
  158. if (!xfs_ilock_nowait(ip, XFS_ILOCK_EXCL))
  159. return EAGAIN;
  160. isize = xfs_ioend_new_eof(ioend);
  161. if (isize) {
  162. ip->i_d.di_size = isize;
  163. xfs_mark_inode_dirty(ip);
  164. }
  165. xfs_iunlock(ip, XFS_ILOCK_EXCL);
  166. return 0;
  167. }
  168. /*
  169. * Schedule IO completion handling on the final put of an ioend.
  170. */
  171. STATIC void
  172. xfs_finish_ioend(
  173. struct xfs_ioend *ioend)
  174. {
  175. if (atomic_dec_and_test(&ioend->io_remaining)) {
  176. if (ioend->io_type == IO_UNWRITTEN)
  177. queue_work(xfsconvertd_workqueue, &ioend->io_work);
  178. else
  179. queue_work(xfsdatad_workqueue, &ioend->io_work);
  180. }
  181. }
  182. /*
  183. * IO write completion.
  184. */
  185. STATIC void
  186. xfs_end_io(
  187. struct work_struct *work)
  188. {
  189. xfs_ioend_t *ioend = container_of(work, xfs_ioend_t, io_work);
  190. struct xfs_inode *ip = XFS_I(ioend->io_inode);
  191. int error = 0;
  192. /*
  193. * For unwritten extents we need to issue transactions to convert a
  194. * range to normal written extens after the data I/O has finished.
  195. */
  196. if (ioend->io_type == IO_UNWRITTEN &&
  197. likely(!ioend->io_error && !XFS_FORCED_SHUTDOWN(ip->i_mount))) {
  198. error = xfs_iomap_write_unwritten(ip, ioend->io_offset,
  199. ioend->io_size);
  200. if (error)
  201. ioend->io_error = error;
  202. }
  203. /*
  204. * We might have to update the on-disk file size after extending
  205. * writes.
  206. */
  207. error = xfs_setfilesize(ioend);
  208. ASSERT(!error || error == EAGAIN);
  209. /*
  210. * If we didn't complete processing of the ioend, requeue it to the
  211. * tail of the workqueue for another attempt later. Otherwise destroy
  212. * it.
  213. */
  214. if (error == EAGAIN) {
  215. atomic_inc(&ioend->io_remaining);
  216. xfs_finish_ioend(ioend);
  217. /* ensure we don't spin on blocked ioends */
  218. delay(1);
  219. } else {
  220. if (ioend->io_iocb)
  221. aio_complete(ioend->io_iocb, ioend->io_result, 0);
  222. xfs_destroy_ioend(ioend);
  223. }
  224. }
  225. /*
  226. * Call IO completion handling in caller context on the final put of an ioend.
  227. */
  228. STATIC void
  229. xfs_finish_ioend_sync(
  230. struct xfs_ioend *ioend)
  231. {
  232. if (atomic_dec_and_test(&ioend->io_remaining))
  233. xfs_end_io(&ioend->io_work);
  234. }
  235. /*
  236. * Allocate and initialise an IO completion structure.
  237. * We need to track unwritten extent write completion here initially.
  238. * We'll need to extend this for updating the ondisk inode size later
  239. * (vs. incore size).
  240. */
  241. STATIC xfs_ioend_t *
  242. xfs_alloc_ioend(
  243. struct inode *inode,
  244. unsigned int type)
  245. {
  246. xfs_ioend_t *ioend;
  247. ioend = mempool_alloc(xfs_ioend_pool, GFP_NOFS);
  248. /*
  249. * Set the count to 1 initially, which will prevent an I/O
  250. * completion callback from happening before we have started
  251. * all the I/O from calling the completion routine too early.
  252. */
  253. atomic_set(&ioend->io_remaining, 1);
  254. ioend->io_error = 0;
  255. ioend->io_list = NULL;
  256. ioend->io_type = type;
  257. ioend->io_inode = inode;
  258. ioend->io_buffer_head = NULL;
  259. ioend->io_buffer_tail = NULL;
  260. atomic_inc(&XFS_I(ioend->io_inode)->i_iocount);
  261. ioend->io_offset = 0;
  262. ioend->io_size = 0;
  263. ioend->io_iocb = NULL;
  264. ioend->io_result = 0;
  265. INIT_WORK(&ioend->io_work, xfs_end_io);
  266. return ioend;
  267. }
  268. STATIC int
  269. xfs_map_blocks(
  270. struct inode *inode,
  271. loff_t offset,
  272. struct xfs_bmbt_irec *imap,
  273. int type,
  274. int nonblocking)
  275. {
  276. struct xfs_inode *ip = XFS_I(inode);
  277. struct xfs_mount *mp = ip->i_mount;
  278. ssize_t count = 1 << inode->i_blkbits;
  279. xfs_fileoff_t offset_fsb, end_fsb;
  280. int error = 0;
  281. int bmapi_flags = XFS_BMAPI_ENTIRE;
  282. int nimaps = 1;
  283. if (XFS_FORCED_SHUTDOWN(mp))
  284. return -XFS_ERROR(EIO);
  285. if (type == IO_UNWRITTEN)
  286. bmapi_flags |= XFS_BMAPI_IGSTATE;
  287. if (!xfs_ilock_nowait(ip, XFS_ILOCK_SHARED)) {
  288. if (nonblocking)
  289. return -XFS_ERROR(EAGAIN);
  290. xfs_ilock(ip, XFS_ILOCK_SHARED);
  291. }
  292. ASSERT(ip->i_d.di_format != XFS_DINODE_FMT_BTREE ||
  293. (ip->i_df.if_flags & XFS_IFEXTENTS));
  294. ASSERT(offset <= mp->m_maxioffset);
  295. if (offset + count > mp->m_maxioffset)
  296. count = mp->m_maxioffset - offset;
  297. end_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + count);
  298. offset_fsb = XFS_B_TO_FSBT(mp, offset);
  299. error = xfs_bmapi(NULL, ip, offset_fsb, end_fsb - offset_fsb,
  300. bmapi_flags, NULL, 0, imap, &nimaps, NULL);
  301. xfs_iunlock(ip, XFS_ILOCK_SHARED);
  302. if (error)
  303. return -XFS_ERROR(error);
  304. if (type == IO_DELALLOC &&
  305. (!nimaps || isnullstartblock(imap->br_startblock))) {
  306. error = xfs_iomap_write_allocate(ip, offset, count, imap);
  307. if (!error)
  308. trace_xfs_map_blocks_alloc(ip, offset, count, type, imap);
  309. return -XFS_ERROR(error);
  310. }
  311. #ifdef DEBUG
  312. if (type == IO_UNWRITTEN) {
  313. ASSERT(nimaps);
  314. ASSERT(imap->br_startblock != HOLESTARTBLOCK);
  315. ASSERT(imap->br_startblock != DELAYSTARTBLOCK);
  316. }
  317. #endif
  318. if (nimaps)
  319. trace_xfs_map_blocks_found(ip, offset, count, type, imap);
  320. return 0;
  321. }
  322. STATIC int
  323. xfs_imap_valid(
  324. struct inode *inode,
  325. struct xfs_bmbt_irec *imap,
  326. xfs_off_t offset)
  327. {
  328. offset >>= inode->i_blkbits;
  329. return offset >= imap->br_startoff &&
  330. offset < imap->br_startoff + imap->br_blockcount;
  331. }
  332. /*
  333. * BIO completion handler for buffered IO.
  334. */
  335. STATIC void
  336. xfs_end_bio(
  337. struct bio *bio,
  338. int error)
  339. {
  340. xfs_ioend_t *ioend = bio->bi_private;
  341. ASSERT(atomic_read(&bio->bi_cnt) >= 1);
  342. ioend->io_error = test_bit(BIO_UPTODATE, &bio->bi_flags) ? 0 : error;
  343. /* Toss bio and pass work off to an xfsdatad thread */
  344. bio->bi_private = NULL;
  345. bio->bi_end_io = NULL;
  346. bio_put(bio);
  347. xfs_finish_ioend(ioend);
  348. }
  349. STATIC void
  350. xfs_submit_ioend_bio(
  351. struct writeback_control *wbc,
  352. xfs_ioend_t *ioend,
  353. struct bio *bio)
  354. {
  355. atomic_inc(&ioend->io_remaining);
  356. bio->bi_private = ioend;
  357. bio->bi_end_io = xfs_end_bio;
  358. /*
  359. * If the I/O is beyond EOF we mark the inode dirty immediately
  360. * but don't update the inode size until I/O completion.
  361. */
  362. if (xfs_ioend_new_eof(ioend))
  363. xfs_mark_inode_dirty(XFS_I(ioend->io_inode));
  364. submit_bio(wbc->sync_mode == WB_SYNC_ALL ?
  365. WRITE_SYNC_PLUG : WRITE, bio);
  366. }
  367. STATIC struct bio *
  368. xfs_alloc_ioend_bio(
  369. struct buffer_head *bh)
  370. {
  371. int nvecs = bio_get_nr_vecs(bh->b_bdev);
  372. struct bio *bio = bio_alloc(GFP_NOIO, nvecs);
  373. ASSERT(bio->bi_private == NULL);
  374. bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
  375. bio->bi_bdev = bh->b_bdev;
  376. return bio;
  377. }
  378. STATIC void
  379. xfs_start_buffer_writeback(
  380. struct buffer_head *bh)
  381. {
  382. ASSERT(buffer_mapped(bh));
  383. ASSERT(buffer_locked(bh));
  384. ASSERT(!buffer_delay(bh));
  385. ASSERT(!buffer_unwritten(bh));
  386. mark_buffer_async_write(bh);
  387. set_buffer_uptodate(bh);
  388. clear_buffer_dirty(bh);
  389. }
  390. STATIC void
  391. xfs_start_page_writeback(
  392. struct page *page,
  393. int clear_dirty,
  394. int buffers)
  395. {
  396. ASSERT(PageLocked(page));
  397. ASSERT(!PageWriteback(page));
  398. if (clear_dirty)
  399. clear_page_dirty_for_io(page);
  400. set_page_writeback(page);
  401. unlock_page(page);
  402. /* If no buffers on the page are to be written, finish it here */
  403. if (!buffers)
  404. end_page_writeback(page);
  405. }
  406. static inline int bio_add_buffer(struct bio *bio, struct buffer_head *bh)
  407. {
  408. return bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
  409. }
  410. /*
  411. * Submit all of the bios for all of the ioends we have saved up, covering the
  412. * initial writepage page and also any probed pages.
  413. *
  414. * Because we may have multiple ioends spanning a page, we need to start
  415. * writeback on all the buffers before we submit them for I/O. If we mark the
  416. * buffers as we got, then we can end up with a page that only has buffers
  417. * marked async write and I/O complete on can occur before we mark the other
  418. * buffers async write.
  419. *
  420. * The end result of this is that we trip a bug in end_page_writeback() because
  421. * we call it twice for the one page as the code in end_buffer_async_write()
  422. * assumes that all buffers on the page are started at the same time.
  423. *
  424. * The fix is two passes across the ioend list - one to start writeback on the
  425. * buffer_heads, and then submit them for I/O on the second pass.
  426. */
  427. STATIC void
  428. xfs_submit_ioend(
  429. struct writeback_control *wbc,
  430. xfs_ioend_t *ioend)
  431. {
  432. xfs_ioend_t *head = ioend;
  433. xfs_ioend_t *next;
  434. struct buffer_head *bh;
  435. struct bio *bio;
  436. sector_t lastblock = 0;
  437. /* Pass 1 - start writeback */
  438. do {
  439. next = ioend->io_list;
  440. for (bh = ioend->io_buffer_head; bh; bh = bh->b_private)
  441. xfs_start_buffer_writeback(bh);
  442. } while ((ioend = next) != NULL);
  443. /* Pass 2 - submit I/O */
  444. ioend = head;
  445. do {
  446. next = ioend->io_list;
  447. bio = NULL;
  448. for (bh = ioend->io_buffer_head; bh; bh = bh->b_private) {
  449. if (!bio) {
  450. retry:
  451. bio = xfs_alloc_ioend_bio(bh);
  452. } else if (bh->b_blocknr != lastblock + 1) {
  453. xfs_submit_ioend_bio(wbc, ioend, bio);
  454. goto retry;
  455. }
  456. if (bio_add_buffer(bio, bh) != bh->b_size) {
  457. xfs_submit_ioend_bio(wbc, ioend, bio);
  458. goto retry;
  459. }
  460. lastblock = bh->b_blocknr;
  461. }
  462. if (bio)
  463. xfs_submit_ioend_bio(wbc, ioend, bio);
  464. xfs_finish_ioend(ioend);
  465. } while ((ioend = next) != NULL);
  466. }
  467. /*
  468. * Cancel submission of all buffer_heads so far in this endio.
  469. * Toss the endio too. Only ever called for the initial page
  470. * in a writepage request, so only ever one page.
  471. */
  472. STATIC void
  473. xfs_cancel_ioend(
  474. xfs_ioend_t *ioend)
  475. {
  476. xfs_ioend_t *next;
  477. struct buffer_head *bh, *next_bh;
  478. do {
  479. next = ioend->io_list;
  480. bh = ioend->io_buffer_head;
  481. do {
  482. next_bh = bh->b_private;
  483. clear_buffer_async_write(bh);
  484. unlock_buffer(bh);
  485. } while ((bh = next_bh) != NULL);
  486. xfs_ioend_wake(XFS_I(ioend->io_inode));
  487. mempool_free(ioend, xfs_ioend_pool);
  488. } while ((ioend = next) != NULL);
  489. }
  490. /*
  491. * Test to see if we've been building up a completion structure for
  492. * earlier buffers -- if so, we try to append to this ioend if we
  493. * can, otherwise we finish off any current ioend and start another.
  494. * Return true if we've finished the given ioend.
  495. */
  496. STATIC void
  497. xfs_add_to_ioend(
  498. struct inode *inode,
  499. struct buffer_head *bh,
  500. xfs_off_t offset,
  501. unsigned int type,
  502. xfs_ioend_t **result,
  503. int need_ioend)
  504. {
  505. xfs_ioend_t *ioend = *result;
  506. if (!ioend || need_ioend || type != ioend->io_type) {
  507. xfs_ioend_t *previous = *result;
  508. ioend = xfs_alloc_ioend(inode, type);
  509. ioend->io_offset = offset;
  510. ioend->io_buffer_head = bh;
  511. ioend->io_buffer_tail = bh;
  512. if (previous)
  513. previous->io_list = ioend;
  514. *result = ioend;
  515. } else {
  516. ioend->io_buffer_tail->b_private = bh;
  517. ioend->io_buffer_tail = bh;
  518. }
  519. bh->b_private = NULL;
  520. ioend->io_size += bh->b_size;
  521. }
  522. STATIC void
  523. xfs_map_buffer(
  524. struct inode *inode,
  525. struct buffer_head *bh,
  526. struct xfs_bmbt_irec *imap,
  527. xfs_off_t offset)
  528. {
  529. sector_t bn;
  530. struct xfs_mount *m = XFS_I(inode)->i_mount;
  531. xfs_off_t iomap_offset = XFS_FSB_TO_B(m, imap->br_startoff);
  532. xfs_daddr_t iomap_bn = xfs_fsb_to_db(XFS_I(inode), imap->br_startblock);
  533. ASSERT(imap->br_startblock != HOLESTARTBLOCK);
  534. ASSERT(imap->br_startblock != DELAYSTARTBLOCK);
  535. bn = (iomap_bn >> (inode->i_blkbits - BBSHIFT)) +
  536. ((offset - iomap_offset) >> inode->i_blkbits);
  537. ASSERT(bn || XFS_IS_REALTIME_INODE(XFS_I(inode)));
  538. bh->b_blocknr = bn;
  539. set_buffer_mapped(bh);
  540. }
  541. STATIC void
  542. xfs_map_at_offset(
  543. struct inode *inode,
  544. struct buffer_head *bh,
  545. struct xfs_bmbt_irec *imap,
  546. xfs_off_t offset)
  547. {
  548. ASSERT(imap->br_startblock != HOLESTARTBLOCK);
  549. ASSERT(imap->br_startblock != DELAYSTARTBLOCK);
  550. lock_buffer(bh);
  551. xfs_map_buffer(inode, bh, imap, offset);
  552. bh->b_bdev = xfs_find_bdev_for_inode(inode);
  553. set_buffer_mapped(bh);
  554. clear_buffer_delay(bh);
  555. clear_buffer_unwritten(bh);
  556. }
  557. /*
  558. * Test if a given page is suitable for writing as part of an unwritten
  559. * or delayed allocate extent.
  560. */
  561. STATIC int
  562. xfs_is_delayed_page(
  563. struct page *page,
  564. unsigned int type)
  565. {
  566. if (PageWriteback(page))
  567. return 0;
  568. if (page->mapping && page_has_buffers(page)) {
  569. struct buffer_head *bh, *head;
  570. int acceptable = 0;
  571. bh = head = page_buffers(page);
  572. do {
  573. if (buffer_unwritten(bh))
  574. acceptable = (type == IO_UNWRITTEN);
  575. else if (buffer_delay(bh))
  576. acceptable = (type == IO_DELALLOC);
  577. else if (buffer_dirty(bh) && buffer_mapped(bh))
  578. acceptable = (type == IO_OVERWRITE);
  579. else
  580. break;
  581. } while ((bh = bh->b_this_page) != head);
  582. if (acceptable)
  583. return 1;
  584. }
  585. return 0;
  586. }
  587. /*
  588. * Allocate & map buffers for page given the extent map. Write it out.
  589. * except for the original page of a writepage, this is called on
  590. * delalloc/unwritten pages only, for the original page it is possible
  591. * that the page has no mapping at all.
  592. */
  593. STATIC int
  594. xfs_convert_page(
  595. struct inode *inode,
  596. struct page *page,
  597. loff_t tindex,
  598. struct xfs_bmbt_irec *imap,
  599. xfs_ioend_t **ioendp,
  600. struct writeback_control *wbc)
  601. {
  602. struct buffer_head *bh, *head;
  603. xfs_off_t end_offset;
  604. unsigned long p_offset;
  605. unsigned int type;
  606. int len, page_dirty;
  607. int count = 0, done = 0, uptodate = 1;
  608. xfs_off_t offset = page_offset(page);
  609. if (page->index != tindex)
  610. goto fail;
  611. if (!trylock_page(page))
  612. goto fail;
  613. if (PageWriteback(page))
  614. goto fail_unlock_page;
  615. if (page->mapping != inode->i_mapping)
  616. goto fail_unlock_page;
  617. if (!xfs_is_delayed_page(page, (*ioendp)->io_type))
  618. goto fail_unlock_page;
  619. /*
  620. * page_dirty is initially a count of buffers on the page before
  621. * EOF and is decremented as we move each into a cleanable state.
  622. *
  623. * Derivation:
  624. *
  625. * End offset is the highest offset that this page should represent.
  626. * If we are on the last page, (end_offset & (PAGE_CACHE_SIZE - 1))
  627. * will evaluate non-zero and be less than PAGE_CACHE_SIZE and
  628. * hence give us the correct page_dirty count. On any other page,
  629. * it will be zero and in that case we need page_dirty to be the
  630. * count of buffers on the page.
  631. */
  632. end_offset = min_t(unsigned long long,
  633. (xfs_off_t)(page->index + 1) << PAGE_CACHE_SHIFT,
  634. i_size_read(inode));
  635. len = 1 << inode->i_blkbits;
  636. p_offset = min_t(unsigned long, end_offset & (PAGE_CACHE_SIZE - 1),
  637. PAGE_CACHE_SIZE);
  638. p_offset = p_offset ? roundup(p_offset, len) : PAGE_CACHE_SIZE;
  639. page_dirty = p_offset / len;
  640. bh = head = page_buffers(page);
  641. do {
  642. if (offset >= end_offset)
  643. break;
  644. if (!buffer_uptodate(bh))
  645. uptodate = 0;
  646. if (!(PageUptodate(page) || buffer_uptodate(bh))) {
  647. done = 1;
  648. continue;
  649. }
  650. if (buffer_unwritten(bh) || buffer_delay(bh) ||
  651. buffer_mapped(bh)) {
  652. if (buffer_unwritten(bh))
  653. type = IO_UNWRITTEN;
  654. else if (buffer_delay(bh))
  655. type = IO_DELALLOC;
  656. else
  657. type = IO_OVERWRITE;
  658. if (!xfs_imap_valid(inode, imap, offset)) {
  659. done = 1;
  660. continue;
  661. }
  662. ASSERT(imap->br_startblock != HOLESTARTBLOCK);
  663. ASSERT(imap->br_startblock != DELAYSTARTBLOCK);
  664. if (type == IO_OVERWRITE)
  665. lock_buffer(bh);
  666. else
  667. xfs_map_at_offset(inode, bh, imap, offset);
  668. xfs_add_to_ioend(inode, bh, offset, type,
  669. ioendp, done);
  670. page_dirty--;
  671. count++;
  672. } else {
  673. done = 1;
  674. }
  675. } while (offset += len, (bh = bh->b_this_page) != head);
  676. if (uptodate && bh == head)
  677. SetPageUptodate(page);
  678. if (count) {
  679. if (--wbc->nr_to_write <= 0 &&
  680. wbc->sync_mode == WB_SYNC_NONE)
  681. done = 1;
  682. }
  683. xfs_start_page_writeback(page, !page_dirty, count);
  684. return done;
  685. fail_unlock_page:
  686. unlock_page(page);
  687. fail:
  688. return 1;
  689. }
  690. /*
  691. * Convert & write out a cluster of pages in the same extent as defined
  692. * by mp and following the start page.
  693. */
  694. STATIC void
  695. xfs_cluster_write(
  696. struct inode *inode,
  697. pgoff_t tindex,
  698. struct xfs_bmbt_irec *imap,
  699. xfs_ioend_t **ioendp,
  700. struct writeback_control *wbc,
  701. pgoff_t tlast)
  702. {
  703. struct pagevec pvec;
  704. int done = 0, i;
  705. pagevec_init(&pvec, 0);
  706. while (!done && tindex <= tlast) {
  707. unsigned len = min_t(pgoff_t, PAGEVEC_SIZE, tlast - tindex + 1);
  708. if (!pagevec_lookup(&pvec, inode->i_mapping, tindex, len))
  709. break;
  710. for (i = 0; i < pagevec_count(&pvec); i++) {
  711. done = xfs_convert_page(inode, pvec.pages[i], tindex++,
  712. imap, ioendp, wbc);
  713. if (done)
  714. break;
  715. }
  716. pagevec_release(&pvec);
  717. cond_resched();
  718. }
  719. }
  720. STATIC void
  721. xfs_vm_invalidatepage(
  722. struct page *page,
  723. unsigned long offset)
  724. {
  725. trace_xfs_invalidatepage(page->mapping->host, page, offset);
  726. block_invalidatepage(page, offset);
  727. }
  728. /*
  729. * If the page has delalloc buffers on it, we need to punch them out before we
  730. * invalidate the page. If we don't, we leave a stale delalloc mapping on the
  731. * inode that can trip a BUG() in xfs_get_blocks() later on if a direct IO read
  732. * is done on that same region - the delalloc extent is returned when none is
  733. * supposed to be there.
  734. *
  735. * We prevent this by truncating away the delalloc regions on the page before
  736. * invalidating it. Because they are delalloc, we can do this without needing a
  737. * transaction. Indeed - if we get ENOSPC errors, we have to be able to do this
  738. * truncation without a transaction as there is no space left for block
  739. * reservation (typically why we see a ENOSPC in writeback).
  740. *
  741. * This is not a performance critical path, so for now just do the punching a
  742. * buffer head at a time.
  743. */
  744. STATIC void
  745. xfs_aops_discard_page(
  746. struct page *page)
  747. {
  748. struct inode *inode = page->mapping->host;
  749. struct xfs_inode *ip = XFS_I(inode);
  750. struct buffer_head *bh, *head;
  751. loff_t offset = page_offset(page);
  752. if (!xfs_is_delayed_page(page, IO_DELALLOC))
  753. goto out_invalidate;
  754. if (XFS_FORCED_SHUTDOWN(ip->i_mount))
  755. goto out_invalidate;
  756. xfs_fs_cmn_err(CE_ALERT, ip->i_mount,
  757. "page discard on page %p, inode 0x%llx, offset %llu.",
  758. page, ip->i_ino, offset);
  759. xfs_ilock(ip, XFS_ILOCK_EXCL);
  760. bh = head = page_buffers(page);
  761. do {
  762. int error;
  763. xfs_fileoff_t start_fsb;
  764. if (!buffer_delay(bh))
  765. goto next_buffer;
  766. start_fsb = XFS_B_TO_FSBT(ip->i_mount, offset);
  767. error = xfs_bmap_punch_delalloc_range(ip, start_fsb, 1);
  768. if (error) {
  769. /* something screwed, just bail */
  770. if (!XFS_FORCED_SHUTDOWN(ip->i_mount)) {
  771. xfs_fs_cmn_err(CE_ALERT, ip->i_mount,
  772. "page discard unable to remove delalloc mapping.");
  773. }
  774. break;
  775. }
  776. next_buffer:
  777. offset += 1 << inode->i_blkbits;
  778. } while ((bh = bh->b_this_page) != head);
  779. xfs_iunlock(ip, XFS_ILOCK_EXCL);
  780. out_invalidate:
  781. xfs_vm_invalidatepage(page, 0);
  782. return;
  783. }
  784. /*
  785. * Write out a dirty page.
  786. *
  787. * For delalloc space on the page we need to allocate space and flush it.
  788. * For unwritten space on the page we need to start the conversion to
  789. * regular allocated space.
  790. * For any other dirty buffer heads on the page we should flush them.
  791. *
  792. * If we detect that a transaction would be required to flush the page, we
  793. * have to check the process flags first, if we are already in a transaction
  794. * or disk I/O during allocations is off, we need to fail the writepage and
  795. * redirty the page.
  796. */
  797. STATIC int
  798. xfs_vm_writepage(
  799. struct page *page,
  800. struct writeback_control *wbc)
  801. {
  802. struct inode *inode = page->mapping->host;
  803. int delalloc, unwritten;
  804. struct buffer_head *bh, *head;
  805. struct xfs_bmbt_irec imap;
  806. xfs_ioend_t *ioend = NULL, *iohead = NULL;
  807. loff_t offset;
  808. unsigned int type;
  809. __uint64_t end_offset;
  810. pgoff_t end_index, last_index;
  811. ssize_t len;
  812. int err, imap_valid = 0, uptodate = 1;
  813. int count = 0;
  814. int nonblocking = 0;
  815. trace_xfs_writepage(inode, page, 0);
  816. ASSERT(page_has_buffers(page));
  817. /*
  818. * Refuse to write the page out if we are called from reclaim context.
  819. *
  820. * This avoids stack overflows when called from deeply used stacks in
  821. * random callers for direct reclaim or memcg reclaim. We explicitly
  822. * allow reclaim from kswapd as the stack usage there is relatively low.
  823. *
  824. * This should really be done by the core VM, but until that happens
  825. * filesystems like XFS, btrfs and ext4 have to take care of this
  826. * by themselves.
  827. */
  828. if ((current->flags & (PF_MEMALLOC|PF_KSWAPD)) == PF_MEMALLOC)
  829. goto redirty;
  830. /*
  831. * We need a transaction if there are delalloc or unwritten buffers
  832. * on the page.
  833. *
  834. * If we need a transaction and the process flags say we are already
  835. * in a transaction, or no IO is allowed then mark the page dirty
  836. * again and leave the page as is.
  837. */
  838. xfs_count_page_state(page, &delalloc, &unwritten);
  839. if ((current->flags & PF_FSTRANS) && (delalloc || unwritten))
  840. goto redirty;
  841. /* Is this page beyond the end of the file? */
  842. offset = i_size_read(inode);
  843. end_index = offset >> PAGE_CACHE_SHIFT;
  844. last_index = (offset - 1) >> PAGE_CACHE_SHIFT;
  845. if (page->index >= end_index) {
  846. if ((page->index >= end_index + 1) ||
  847. !(i_size_read(inode) & (PAGE_CACHE_SIZE - 1))) {
  848. unlock_page(page);
  849. return 0;
  850. }
  851. }
  852. end_offset = min_t(unsigned long long,
  853. (xfs_off_t)(page->index + 1) << PAGE_CACHE_SHIFT,
  854. offset);
  855. len = 1 << inode->i_blkbits;
  856. bh = head = page_buffers(page);
  857. offset = page_offset(page);
  858. type = IO_OVERWRITE;
  859. if (wbc->sync_mode == WB_SYNC_NONE && wbc->nonblocking)
  860. nonblocking = 1;
  861. do {
  862. int new_ioend = 0;
  863. if (offset >= end_offset)
  864. break;
  865. if (!buffer_uptodate(bh))
  866. uptodate = 0;
  867. /*
  868. * set_page_dirty dirties all buffers in a page, independent
  869. * of their state. The dirty state however is entirely
  870. * meaningless for holes (!mapped && uptodate), so skip
  871. * buffers covering holes here.
  872. */
  873. if (!buffer_mapped(bh) && buffer_uptodate(bh)) {
  874. imap_valid = 0;
  875. continue;
  876. }
  877. if (buffer_unwritten(bh)) {
  878. if (type != IO_UNWRITTEN) {
  879. type = IO_UNWRITTEN;
  880. imap_valid = 0;
  881. }
  882. } else if (buffer_delay(bh)) {
  883. if (type != IO_DELALLOC) {
  884. type = IO_DELALLOC;
  885. imap_valid = 0;
  886. }
  887. } else if (buffer_uptodate(bh)) {
  888. if (type != IO_OVERWRITE) {
  889. type = IO_OVERWRITE;
  890. imap_valid = 0;
  891. }
  892. } else {
  893. if (PageUptodate(page)) {
  894. ASSERT(buffer_mapped(bh));
  895. imap_valid = 0;
  896. }
  897. continue;
  898. }
  899. if (imap_valid)
  900. imap_valid = xfs_imap_valid(inode, &imap, offset);
  901. if (!imap_valid) {
  902. /*
  903. * If we didn't have a valid mapping then we need to
  904. * put the new mapping into a separate ioend structure.
  905. * This ensures non-contiguous extents always have
  906. * separate ioends, which is particularly important
  907. * for unwritten extent conversion at I/O completion
  908. * time.
  909. */
  910. new_ioend = 1;
  911. err = xfs_map_blocks(inode, offset, &imap, type,
  912. nonblocking);
  913. if (err)
  914. goto error;
  915. imap_valid = xfs_imap_valid(inode, &imap, offset);
  916. }
  917. if (imap_valid) {
  918. if (type == IO_OVERWRITE)
  919. lock_buffer(bh);
  920. else
  921. xfs_map_at_offset(inode, bh, &imap, offset);
  922. xfs_add_to_ioend(inode, bh, offset, type, &ioend,
  923. new_ioend);
  924. count++;
  925. }
  926. if (!iohead)
  927. iohead = ioend;
  928. } while (offset += len, ((bh = bh->b_this_page) != head));
  929. if (uptodate && bh == head)
  930. SetPageUptodate(page);
  931. xfs_start_page_writeback(page, 1, count);
  932. if (ioend && imap_valid) {
  933. xfs_off_t end_index;
  934. end_index = imap.br_startoff + imap.br_blockcount;
  935. /* to bytes */
  936. end_index <<= inode->i_blkbits;
  937. /* to pages */
  938. end_index = (end_index - 1) >> PAGE_CACHE_SHIFT;
  939. /* check against file size */
  940. if (end_index > last_index)
  941. end_index = last_index;
  942. xfs_cluster_write(inode, page->index + 1, &imap, &ioend,
  943. wbc, end_index);
  944. }
  945. if (iohead)
  946. xfs_submit_ioend(wbc, iohead);
  947. return 0;
  948. error:
  949. if (iohead)
  950. xfs_cancel_ioend(iohead);
  951. if (err == -EAGAIN)
  952. goto redirty;
  953. xfs_aops_discard_page(page);
  954. ClearPageUptodate(page);
  955. unlock_page(page);
  956. return err;
  957. redirty:
  958. redirty_page_for_writepage(wbc, page);
  959. unlock_page(page);
  960. return 0;
  961. }
  962. STATIC int
  963. xfs_vm_writepages(
  964. struct address_space *mapping,
  965. struct writeback_control *wbc)
  966. {
  967. xfs_iflags_clear(XFS_I(mapping->host), XFS_ITRUNCATED);
  968. return generic_writepages(mapping, wbc);
  969. }
  970. /*
  971. * Called to move a page into cleanable state - and from there
  972. * to be released. The page should already be clean. We always
  973. * have buffer heads in this call.
  974. *
  975. * Returns 1 if the page is ok to release, 0 otherwise.
  976. */
  977. STATIC int
  978. xfs_vm_releasepage(
  979. struct page *page,
  980. gfp_t gfp_mask)
  981. {
  982. int delalloc, unwritten;
  983. trace_xfs_releasepage(page->mapping->host, page, 0);
  984. xfs_count_page_state(page, &delalloc, &unwritten);
  985. if (WARN_ON(delalloc))
  986. return 0;
  987. if (WARN_ON(unwritten))
  988. return 0;
  989. return try_to_free_buffers(page);
  990. }
  991. STATIC int
  992. __xfs_get_blocks(
  993. struct inode *inode,
  994. sector_t iblock,
  995. struct buffer_head *bh_result,
  996. int create,
  997. int direct)
  998. {
  999. struct xfs_inode *ip = XFS_I(inode);
  1000. struct xfs_mount *mp = ip->i_mount;
  1001. xfs_fileoff_t offset_fsb, end_fsb;
  1002. int error = 0;
  1003. int lockmode = 0;
  1004. struct xfs_bmbt_irec imap;
  1005. int nimaps = 1;
  1006. xfs_off_t offset;
  1007. ssize_t size;
  1008. int new = 0;
  1009. if (XFS_FORCED_SHUTDOWN(mp))
  1010. return -XFS_ERROR(EIO);
  1011. offset = (xfs_off_t)iblock << inode->i_blkbits;
  1012. ASSERT(bh_result->b_size >= (1 << inode->i_blkbits));
  1013. size = bh_result->b_size;
  1014. if (!create && direct && offset >= i_size_read(inode))
  1015. return 0;
  1016. if (create) {
  1017. lockmode = XFS_ILOCK_EXCL;
  1018. xfs_ilock(ip, lockmode);
  1019. } else {
  1020. lockmode = xfs_ilock_map_shared(ip);
  1021. }
  1022. ASSERT(offset <= mp->m_maxioffset);
  1023. if (offset + size > mp->m_maxioffset)
  1024. size = mp->m_maxioffset - offset;
  1025. end_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + size);
  1026. offset_fsb = XFS_B_TO_FSBT(mp, offset);
  1027. error = xfs_bmapi(NULL, ip, offset_fsb, end_fsb - offset_fsb,
  1028. XFS_BMAPI_ENTIRE, NULL, 0, &imap, &nimaps, NULL);
  1029. if (error)
  1030. goto out_unlock;
  1031. if (create &&
  1032. (!nimaps ||
  1033. (imap.br_startblock == HOLESTARTBLOCK ||
  1034. imap.br_startblock == DELAYSTARTBLOCK))) {
  1035. if (direct) {
  1036. error = xfs_iomap_write_direct(ip, offset, size,
  1037. &imap, nimaps);
  1038. } else {
  1039. error = xfs_iomap_write_delay(ip, offset, size, &imap);
  1040. }
  1041. if (error)
  1042. goto out_unlock;
  1043. trace_xfs_get_blocks_alloc(ip, offset, size, 0, &imap);
  1044. } else if (nimaps) {
  1045. trace_xfs_get_blocks_found(ip, offset, size, 0, &imap);
  1046. } else {
  1047. trace_xfs_get_blocks_notfound(ip, offset, size);
  1048. goto out_unlock;
  1049. }
  1050. xfs_iunlock(ip, lockmode);
  1051. if (imap.br_startblock != HOLESTARTBLOCK &&
  1052. imap.br_startblock != DELAYSTARTBLOCK) {
  1053. /*
  1054. * For unwritten extents do not report a disk address on
  1055. * the read case (treat as if we're reading into a hole).
  1056. */
  1057. if (create || !ISUNWRITTEN(&imap))
  1058. xfs_map_buffer(inode, bh_result, &imap, offset);
  1059. if (create && ISUNWRITTEN(&imap)) {
  1060. if (direct)
  1061. bh_result->b_private = inode;
  1062. set_buffer_unwritten(bh_result);
  1063. }
  1064. }
  1065. /*
  1066. * If this is a realtime file, data may be on a different device.
  1067. * to that pointed to from the buffer_head b_bdev currently.
  1068. */
  1069. bh_result->b_bdev = xfs_find_bdev_for_inode(inode);
  1070. /*
  1071. * If we previously allocated a block out beyond eof and we are now
  1072. * coming back to use it then we will need to flag it as new even if it
  1073. * has a disk address.
  1074. *
  1075. * With sub-block writes into unwritten extents we also need to mark
  1076. * the buffer as new so that the unwritten parts of the buffer gets
  1077. * correctly zeroed.
  1078. */
  1079. if (create &&
  1080. ((!buffer_mapped(bh_result) && !buffer_uptodate(bh_result)) ||
  1081. (offset >= i_size_read(inode)) ||
  1082. (new || ISUNWRITTEN(&imap))))
  1083. set_buffer_new(bh_result);
  1084. if (imap.br_startblock == DELAYSTARTBLOCK) {
  1085. BUG_ON(direct);
  1086. if (create) {
  1087. set_buffer_uptodate(bh_result);
  1088. set_buffer_mapped(bh_result);
  1089. set_buffer_delay(bh_result);
  1090. }
  1091. }
  1092. /*
  1093. * If this is O_DIRECT or the mpage code calling tell them how large
  1094. * the mapping is, so that we can avoid repeated get_blocks calls.
  1095. */
  1096. if (direct || size > (1 << inode->i_blkbits)) {
  1097. xfs_off_t mapping_size;
  1098. mapping_size = imap.br_startoff + imap.br_blockcount - iblock;
  1099. mapping_size <<= inode->i_blkbits;
  1100. ASSERT(mapping_size > 0);
  1101. if (mapping_size > size)
  1102. mapping_size = size;
  1103. if (mapping_size > LONG_MAX)
  1104. mapping_size = LONG_MAX;
  1105. bh_result->b_size = mapping_size;
  1106. }
  1107. return 0;
  1108. out_unlock:
  1109. xfs_iunlock(ip, lockmode);
  1110. return -error;
  1111. }
  1112. int
  1113. xfs_get_blocks(
  1114. struct inode *inode,
  1115. sector_t iblock,
  1116. struct buffer_head *bh_result,
  1117. int create)
  1118. {
  1119. return __xfs_get_blocks(inode, iblock, bh_result, create, 0);
  1120. }
  1121. STATIC int
  1122. xfs_get_blocks_direct(
  1123. struct inode *inode,
  1124. sector_t iblock,
  1125. struct buffer_head *bh_result,
  1126. int create)
  1127. {
  1128. return __xfs_get_blocks(inode, iblock, bh_result, create, 1);
  1129. }
  1130. /*
  1131. * Complete a direct I/O write request.
  1132. *
  1133. * If the private argument is non-NULL __xfs_get_blocks signals us that we
  1134. * need to issue a transaction to convert the range from unwritten to written
  1135. * extents. In case this is regular synchronous I/O we just call xfs_end_io
  1136. * to do this and we are done. But in case this was a successfull AIO
  1137. * request this handler is called from interrupt context, from which we
  1138. * can't start transactions. In that case offload the I/O completion to
  1139. * the workqueues we also use for buffered I/O completion.
  1140. */
  1141. STATIC void
  1142. xfs_end_io_direct_write(
  1143. struct kiocb *iocb,
  1144. loff_t offset,
  1145. ssize_t size,
  1146. void *private,
  1147. int ret,
  1148. bool is_async)
  1149. {
  1150. struct xfs_ioend *ioend = iocb->private;
  1151. /*
  1152. * blockdev_direct_IO can return an error even after the I/O
  1153. * completion handler was called. Thus we need to protect
  1154. * against double-freeing.
  1155. */
  1156. iocb->private = NULL;
  1157. ioend->io_offset = offset;
  1158. ioend->io_size = size;
  1159. if (private && size > 0)
  1160. ioend->io_type = IO_UNWRITTEN;
  1161. if (is_async) {
  1162. /*
  1163. * If we are converting an unwritten extent we need to delay
  1164. * the AIO completion until after the unwrittent extent
  1165. * conversion has completed, otherwise do it ASAP.
  1166. */
  1167. if (ioend->io_type == IO_UNWRITTEN) {
  1168. ioend->io_iocb = iocb;
  1169. ioend->io_result = ret;
  1170. } else {
  1171. aio_complete(iocb, ret, 0);
  1172. }
  1173. xfs_finish_ioend(ioend);
  1174. } else {
  1175. xfs_finish_ioend_sync(ioend);
  1176. }
  1177. }
  1178. STATIC ssize_t
  1179. xfs_vm_direct_IO(
  1180. int rw,
  1181. struct kiocb *iocb,
  1182. const struct iovec *iov,
  1183. loff_t offset,
  1184. unsigned long nr_segs)
  1185. {
  1186. struct inode *inode = iocb->ki_filp->f_mapping->host;
  1187. struct block_device *bdev = xfs_find_bdev_for_inode(inode);
  1188. ssize_t ret;
  1189. if (rw & WRITE) {
  1190. iocb->private = xfs_alloc_ioend(inode, IO_DIRECT);
  1191. ret = __blockdev_direct_IO(rw, iocb, inode, bdev, iov,
  1192. offset, nr_segs,
  1193. xfs_get_blocks_direct,
  1194. xfs_end_io_direct_write, NULL, 0);
  1195. if (ret != -EIOCBQUEUED && iocb->private)
  1196. xfs_destroy_ioend(iocb->private);
  1197. } else {
  1198. ret = __blockdev_direct_IO(rw, iocb, inode, bdev, iov,
  1199. offset, nr_segs,
  1200. xfs_get_blocks_direct,
  1201. NULL, NULL, 0);
  1202. }
  1203. return ret;
  1204. }
  1205. STATIC void
  1206. xfs_vm_write_failed(
  1207. struct address_space *mapping,
  1208. loff_t to)
  1209. {
  1210. struct inode *inode = mapping->host;
  1211. if (to > inode->i_size) {
  1212. /*
  1213. * punch out the delalloc blocks we have already allocated. We
  1214. * don't call xfs_setattr() to do this as we may be in the
  1215. * middle of a multi-iovec write and so the vfs inode->i_size
  1216. * will not match the xfs ip->i_size and so it will zero too
  1217. * much. Hence we jus truncate the page cache to zero what is
  1218. * necessary and punch the delalloc blocks directly.
  1219. */
  1220. struct xfs_inode *ip = XFS_I(inode);
  1221. xfs_fileoff_t start_fsb;
  1222. xfs_fileoff_t end_fsb;
  1223. int error;
  1224. truncate_pagecache(inode, to, inode->i_size);
  1225. /*
  1226. * Check if there are any blocks that are outside of i_size
  1227. * that need to be trimmed back.
  1228. */
  1229. start_fsb = XFS_B_TO_FSB(ip->i_mount, inode->i_size) + 1;
  1230. end_fsb = XFS_B_TO_FSB(ip->i_mount, to);
  1231. if (end_fsb <= start_fsb)
  1232. return;
  1233. xfs_ilock(ip, XFS_ILOCK_EXCL);
  1234. error = xfs_bmap_punch_delalloc_range(ip, start_fsb,
  1235. end_fsb - start_fsb);
  1236. if (error) {
  1237. /* something screwed, just bail */
  1238. if (!XFS_FORCED_SHUTDOWN(ip->i_mount)) {
  1239. xfs_fs_cmn_err(CE_ALERT, ip->i_mount,
  1240. "xfs_vm_write_failed: unable to clean up ino %lld",
  1241. ip->i_ino);
  1242. }
  1243. }
  1244. xfs_iunlock(ip, XFS_ILOCK_EXCL);
  1245. }
  1246. }
  1247. STATIC int
  1248. xfs_vm_write_begin(
  1249. struct file *file,
  1250. struct address_space *mapping,
  1251. loff_t pos,
  1252. unsigned len,
  1253. unsigned flags,
  1254. struct page **pagep,
  1255. void **fsdata)
  1256. {
  1257. int ret;
  1258. ret = block_write_begin(mapping, pos, len, flags | AOP_FLAG_NOFS,
  1259. pagep, xfs_get_blocks);
  1260. if (unlikely(ret))
  1261. xfs_vm_write_failed(mapping, pos + len);
  1262. return ret;
  1263. }
  1264. STATIC int
  1265. xfs_vm_write_end(
  1266. struct file *file,
  1267. struct address_space *mapping,
  1268. loff_t pos,
  1269. unsigned len,
  1270. unsigned copied,
  1271. struct page *page,
  1272. void *fsdata)
  1273. {
  1274. int ret;
  1275. ret = generic_write_end(file, mapping, pos, len, copied, page, fsdata);
  1276. if (unlikely(ret < len))
  1277. xfs_vm_write_failed(mapping, pos + len);
  1278. return ret;
  1279. }
  1280. STATIC sector_t
  1281. xfs_vm_bmap(
  1282. struct address_space *mapping,
  1283. sector_t block)
  1284. {
  1285. struct inode *inode = (struct inode *)mapping->host;
  1286. struct xfs_inode *ip = XFS_I(inode);
  1287. trace_xfs_vm_bmap(XFS_I(inode));
  1288. xfs_ilock(ip, XFS_IOLOCK_SHARED);
  1289. xfs_flush_pages(ip, (xfs_off_t)0, -1, 0, FI_REMAPF);
  1290. xfs_iunlock(ip, XFS_IOLOCK_SHARED);
  1291. return generic_block_bmap(mapping, block, xfs_get_blocks);
  1292. }
  1293. STATIC int
  1294. xfs_vm_readpage(
  1295. struct file *unused,
  1296. struct page *page)
  1297. {
  1298. return mpage_readpage(page, xfs_get_blocks);
  1299. }
  1300. STATIC int
  1301. xfs_vm_readpages(
  1302. struct file *unused,
  1303. struct address_space *mapping,
  1304. struct list_head *pages,
  1305. unsigned nr_pages)
  1306. {
  1307. return mpage_readpages(mapping, pages, nr_pages, xfs_get_blocks);
  1308. }
  1309. const struct address_space_operations xfs_address_space_operations = {
  1310. .readpage = xfs_vm_readpage,
  1311. .readpages = xfs_vm_readpages,
  1312. .writepage = xfs_vm_writepage,
  1313. .writepages = xfs_vm_writepages,
  1314. .sync_page = block_sync_page,
  1315. .releasepage = xfs_vm_releasepage,
  1316. .invalidatepage = xfs_vm_invalidatepage,
  1317. .write_begin = xfs_vm_write_begin,
  1318. .write_end = xfs_vm_write_end,
  1319. .bmap = xfs_vm_bmap,
  1320. .direct_IO = xfs_vm_direct_IO,
  1321. .migratepage = buffer_migrate_page,
  1322. .is_partially_uptodate = block_is_partially_uptodate,
  1323. .error_remove_page = generic_error_remove_page,
  1324. };