write.c 45 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780
  1. /*
  2. * linux/fs/nfs/write.c
  3. *
  4. * Write file data over NFS.
  5. *
  6. * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de>
  7. */
  8. #include <linux/types.h>
  9. #include <linux/slab.h>
  10. #include <linux/mm.h>
  11. #include <linux/pagemap.h>
  12. #include <linux/file.h>
  13. #include <linux/writeback.h>
  14. #include <linux/swap.h>
  15. #include <linux/migrate.h>
  16. #include <linux/sunrpc/clnt.h>
  17. #include <linux/nfs_fs.h>
  18. #include <linux/nfs_mount.h>
  19. #include <linux/nfs_page.h>
  20. #include <linux/backing-dev.h>
  21. #include <asm/uaccess.h>
  22. #include "delegation.h"
  23. #include "internal.h"
  24. #include "iostat.h"
  25. #include "nfs4_fs.h"
  26. #include "fscache.h"
  27. #include "pnfs.h"
  28. #define NFSDBG_FACILITY NFSDBG_PAGECACHE
  29. #define MIN_POOL_WRITE (32)
  30. #define MIN_POOL_COMMIT (4)
  31. /*
  32. * Local function declarations
  33. */
  34. static void nfs_pageio_init_write(struct nfs_pageio_descriptor *desc,
  35. struct inode *inode, int ioflags);
  36. static void nfs_redirty_request(struct nfs_page *req);
  37. static const struct rpc_call_ops nfs_write_partial_ops;
  38. static const struct rpc_call_ops nfs_write_full_ops;
  39. static const struct rpc_call_ops nfs_commit_ops;
  40. static struct kmem_cache *nfs_wdata_cachep;
  41. static mempool_t *nfs_wdata_mempool;
  42. static mempool_t *nfs_commit_mempool;
  43. struct nfs_write_data *nfs_commitdata_alloc(void)
  44. {
  45. struct nfs_write_data *p = mempool_alloc(nfs_commit_mempool, GFP_NOFS);
  46. if (p) {
  47. memset(p, 0, sizeof(*p));
  48. INIT_LIST_HEAD(&p->pages);
  49. }
  50. return p;
  51. }
  52. EXPORT_SYMBOL_GPL(nfs_commitdata_alloc);
  53. void nfs_commit_free(struct nfs_write_data *p)
  54. {
  55. if (p && (p->pagevec != &p->page_array[0]))
  56. kfree(p->pagevec);
  57. mempool_free(p, nfs_commit_mempool);
  58. }
  59. EXPORT_SYMBOL_GPL(nfs_commit_free);
  60. struct nfs_write_data *nfs_writedata_alloc(unsigned int pagecount)
  61. {
  62. struct nfs_write_data *p = mempool_alloc(nfs_wdata_mempool, GFP_NOFS);
  63. if (p) {
  64. memset(p, 0, sizeof(*p));
  65. INIT_LIST_HEAD(&p->pages);
  66. p->npages = pagecount;
  67. if (pagecount <= ARRAY_SIZE(p->page_array))
  68. p->pagevec = p->page_array;
  69. else {
  70. p->pagevec = kcalloc(pagecount, sizeof(struct page *), GFP_NOFS);
  71. if (!p->pagevec) {
  72. mempool_free(p, nfs_wdata_mempool);
  73. p = NULL;
  74. }
  75. }
  76. }
  77. return p;
  78. }
  79. void nfs_writedata_free(struct nfs_write_data *p)
  80. {
  81. if (p && (p->pagevec != &p->page_array[0]))
  82. kfree(p->pagevec);
  83. mempool_free(p, nfs_wdata_mempool);
  84. }
  85. void nfs_writedata_release(struct nfs_write_data *wdata)
  86. {
  87. put_lseg(wdata->lseg);
  88. put_nfs_open_context(wdata->args.context);
  89. nfs_writedata_free(wdata);
  90. }
  91. static void nfs_context_set_write_error(struct nfs_open_context *ctx, int error)
  92. {
  93. ctx->error = error;
  94. smp_wmb();
  95. set_bit(NFS_CONTEXT_ERROR_WRITE, &ctx->flags);
  96. }
  97. static struct nfs_page *nfs_page_find_request_locked(struct page *page)
  98. {
  99. struct nfs_page *req = NULL;
  100. if (PagePrivate(page)) {
  101. req = (struct nfs_page *)page_private(page);
  102. if (req != NULL)
  103. kref_get(&req->wb_kref);
  104. }
  105. return req;
  106. }
  107. static struct nfs_page *nfs_page_find_request(struct page *page)
  108. {
  109. struct inode *inode = page->mapping->host;
  110. struct nfs_page *req = NULL;
  111. spin_lock(&inode->i_lock);
  112. req = nfs_page_find_request_locked(page);
  113. spin_unlock(&inode->i_lock);
  114. return req;
  115. }
  116. /* Adjust the file length if we're writing beyond the end */
  117. static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int count)
  118. {
  119. struct inode *inode = page->mapping->host;
  120. loff_t end, i_size;
  121. pgoff_t end_index;
  122. spin_lock(&inode->i_lock);
  123. i_size = i_size_read(inode);
  124. end_index = (i_size - 1) >> PAGE_CACHE_SHIFT;
  125. if (i_size > 0 && page->index < end_index)
  126. goto out;
  127. end = ((loff_t)page->index << PAGE_CACHE_SHIFT) + ((loff_t)offset+count);
  128. if (i_size >= end)
  129. goto out;
  130. i_size_write(inode, end);
  131. nfs_inc_stats(inode, NFSIOS_EXTENDWRITE);
  132. out:
  133. spin_unlock(&inode->i_lock);
  134. }
  135. /* A writeback failed: mark the page as bad, and invalidate the page cache */
  136. static void nfs_set_pageerror(struct page *page)
  137. {
  138. SetPageError(page);
  139. nfs_zap_mapping(page->mapping->host, page->mapping);
  140. }
  141. /* We can set the PG_uptodate flag if we see that a write request
  142. * covers the full page.
  143. */
  144. static void nfs_mark_uptodate(struct page *page, unsigned int base, unsigned int count)
  145. {
  146. if (PageUptodate(page))
  147. return;
  148. if (base != 0)
  149. return;
  150. if (count != nfs_page_length(page))
  151. return;
  152. SetPageUptodate(page);
  153. }
  154. static int wb_priority(struct writeback_control *wbc)
  155. {
  156. if (wbc->for_reclaim)
  157. return FLUSH_HIGHPRI | FLUSH_STABLE;
  158. if (wbc->for_kupdate || wbc->for_background)
  159. return FLUSH_LOWPRI | FLUSH_COND_STABLE;
  160. return FLUSH_COND_STABLE;
  161. }
  162. /*
  163. * NFS congestion control
  164. */
  165. int nfs_congestion_kb;
  166. #define NFS_CONGESTION_ON_THRESH (nfs_congestion_kb >> (PAGE_SHIFT-10))
  167. #define NFS_CONGESTION_OFF_THRESH \
  168. (NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2))
  169. static int nfs_set_page_writeback(struct page *page)
  170. {
  171. int ret = test_set_page_writeback(page);
  172. if (!ret) {
  173. struct inode *inode = page->mapping->host;
  174. struct nfs_server *nfss = NFS_SERVER(inode);
  175. page_cache_get(page);
  176. if (atomic_long_inc_return(&nfss->writeback) >
  177. NFS_CONGESTION_ON_THRESH) {
  178. set_bdi_congested(&nfss->backing_dev_info,
  179. BLK_RW_ASYNC);
  180. }
  181. }
  182. return ret;
  183. }
  184. static void nfs_end_page_writeback(struct page *page)
  185. {
  186. struct inode *inode = page->mapping->host;
  187. struct nfs_server *nfss = NFS_SERVER(inode);
  188. end_page_writeback(page);
  189. page_cache_release(page);
  190. if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH)
  191. clear_bdi_congested(&nfss->backing_dev_info, BLK_RW_ASYNC);
  192. }
  193. static struct nfs_page *nfs_find_and_lock_request(struct page *page, bool nonblock)
  194. {
  195. struct inode *inode = page->mapping->host;
  196. struct nfs_page *req;
  197. int ret;
  198. spin_lock(&inode->i_lock);
  199. for (;;) {
  200. req = nfs_page_find_request_locked(page);
  201. if (req == NULL)
  202. break;
  203. if (nfs_set_page_tag_locked(req))
  204. break;
  205. /* Note: If we hold the page lock, as is the case in nfs_writepage,
  206. * then the call to nfs_set_page_tag_locked() will always
  207. * succeed provided that someone hasn't already marked the
  208. * request as dirty (in which case we don't care).
  209. */
  210. spin_unlock(&inode->i_lock);
  211. if (!nonblock)
  212. ret = nfs_wait_on_request(req);
  213. else
  214. ret = -EAGAIN;
  215. nfs_release_request(req);
  216. if (ret != 0)
  217. return ERR_PTR(ret);
  218. spin_lock(&inode->i_lock);
  219. }
  220. spin_unlock(&inode->i_lock);
  221. return req;
  222. }
  223. /*
  224. * Find an associated nfs write request, and prepare to flush it out
  225. * May return an error if the user signalled nfs_wait_on_request().
  226. */
  227. static int nfs_page_async_flush(struct nfs_pageio_descriptor *pgio,
  228. struct page *page, bool nonblock)
  229. {
  230. struct nfs_page *req;
  231. int ret = 0;
  232. req = nfs_find_and_lock_request(page, nonblock);
  233. if (!req)
  234. goto out;
  235. ret = PTR_ERR(req);
  236. if (IS_ERR(req))
  237. goto out;
  238. ret = nfs_set_page_writeback(page);
  239. BUG_ON(ret != 0);
  240. BUG_ON(test_bit(PG_CLEAN, &req->wb_flags));
  241. if (!nfs_pageio_add_request(pgio, req)) {
  242. nfs_redirty_request(req);
  243. ret = pgio->pg_error;
  244. }
  245. out:
  246. return ret;
  247. }
  248. static int nfs_do_writepage(struct page *page, struct writeback_control *wbc, struct nfs_pageio_descriptor *pgio)
  249. {
  250. struct inode *inode = page->mapping->host;
  251. int ret;
  252. nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE);
  253. nfs_add_stats(inode, NFSIOS_WRITEPAGES, 1);
  254. nfs_pageio_cond_complete(pgio, page->index);
  255. ret = nfs_page_async_flush(pgio, page, wbc->sync_mode == WB_SYNC_NONE);
  256. if (ret == -EAGAIN) {
  257. redirty_page_for_writepage(wbc, page);
  258. ret = 0;
  259. }
  260. return ret;
  261. }
  262. /*
  263. * Write an mmapped page to the server.
  264. */
  265. static int nfs_writepage_locked(struct page *page, struct writeback_control *wbc)
  266. {
  267. struct nfs_pageio_descriptor pgio;
  268. int err;
  269. nfs_pageio_init_write(&pgio, page->mapping->host, wb_priority(wbc));
  270. err = nfs_do_writepage(page, wbc, &pgio);
  271. nfs_pageio_complete(&pgio);
  272. if (err < 0)
  273. return err;
  274. if (pgio.pg_error < 0)
  275. return pgio.pg_error;
  276. return 0;
  277. }
  278. int nfs_writepage(struct page *page, struct writeback_control *wbc)
  279. {
  280. int ret;
  281. ret = nfs_writepage_locked(page, wbc);
  282. unlock_page(page);
  283. return ret;
  284. }
  285. static int nfs_writepages_callback(struct page *page, struct writeback_control *wbc, void *data)
  286. {
  287. int ret;
  288. ret = nfs_do_writepage(page, wbc, data);
  289. unlock_page(page);
  290. return ret;
  291. }
  292. int nfs_writepages(struct address_space *mapping, struct writeback_control *wbc)
  293. {
  294. struct inode *inode = mapping->host;
  295. unsigned long *bitlock = &NFS_I(inode)->flags;
  296. struct nfs_pageio_descriptor pgio;
  297. int err;
  298. /* Stop dirtying of new pages while we sync */
  299. err = wait_on_bit_lock(bitlock, NFS_INO_FLUSHING,
  300. nfs_wait_bit_killable, TASK_KILLABLE);
  301. if (err)
  302. goto out_err;
  303. nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGES);
  304. nfs_pageio_init_write(&pgio, inode, wb_priority(wbc));
  305. err = write_cache_pages(mapping, wbc, nfs_writepages_callback, &pgio);
  306. nfs_pageio_complete(&pgio);
  307. clear_bit_unlock(NFS_INO_FLUSHING, bitlock);
  308. smp_mb__after_clear_bit();
  309. wake_up_bit(bitlock, NFS_INO_FLUSHING);
  310. if (err < 0)
  311. goto out_err;
  312. err = pgio.pg_error;
  313. if (err < 0)
  314. goto out_err;
  315. return 0;
  316. out_err:
  317. return err;
  318. }
  319. /*
  320. * Insert a write request into an inode
  321. */
  322. static int nfs_inode_add_request(struct inode *inode, struct nfs_page *req)
  323. {
  324. struct nfs_inode *nfsi = NFS_I(inode);
  325. int error;
  326. error = radix_tree_preload(GFP_NOFS);
  327. if (error != 0)
  328. goto out;
  329. /* Lock the request! */
  330. nfs_lock_request_dontget(req);
  331. spin_lock(&inode->i_lock);
  332. error = radix_tree_insert(&nfsi->nfs_page_tree, req->wb_index, req);
  333. BUG_ON(error);
  334. if (!nfsi->npages && nfs_have_delegation(inode, FMODE_WRITE))
  335. inode->i_version++;
  336. set_bit(PG_MAPPED, &req->wb_flags);
  337. SetPagePrivate(req->wb_page);
  338. set_page_private(req->wb_page, (unsigned long)req);
  339. nfsi->npages++;
  340. kref_get(&req->wb_kref);
  341. radix_tree_tag_set(&nfsi->nfs_page_tree, req->wb_index,
  342. NFS_PAGE_TAG_LOCKED);
  343. spin_unlock(&inode->i_lock);
  344. radix_tree_preload_end();
  345. out:
  346. return error;
  347. }
  348. /*
  349. * Remove a write request from an inode
  350. */
  351. static void nfs_inode_remove_request(struct nfs_page *req)
  352. {
  353. struct inode *inode = req->wb_context->dentry->d_inode;
  354. struct nfs_inode *nfsi = NFS_I(inode);
  355. BUG_ON (!NFS_WBACK_BUSY(req));
  356. spin_lock(&inode->i_lock);
  357. set_page_private(req->wb_page, 0);
  358. ClearPagePrivate(req->wb_page);
  359. clear_bit(PG_MAPPED, &req->wb_flags);
  360. radix_tree_delete(&nfsi->nfs_page_tree, req->wb_index);
  361. nfsi->npages--;
  362. spin_unlock(&inode->i_lock);
  363. nfs_release_request(req);
  364. }
  365. static void
  366. nfs_mark_request_dirty(struct nfs_page *req)
  367. {
  368. __set_page_dirty_nobuffers(req->wb_page);
  369. }
  370. #if defined(CONFIG_NFS_V3) || defined(CONFIG_NFS_V4)
  371. /*
  372. * Add a request to the inode's commit list.
  373. */
  374. static void
  375. nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg)
  376. {
  377. struct inode *inode = req->wb_context->dentry->d_inode;
  378. struct nfs_inode *nfsi = NFS_I(inode);
  379. spin_lock(&inode->i_lock);
  380. set_bit(PG_CLEAN, &(req)->wb_flags);
  381. radix_tree_tag_set(&nfsi->nfs_page_tree,
  382. req->wb_index,
  383. NFS_PAGE_TAG_COMMIT);
  384. nfsi->ncommit++;
  385. spin_unlock(&inode->i_lock);
  386. pnfs_mark_request_commit(req, lseg);
  387. inc_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
  388. inc_bdi_stat(req->wb_page->mapping->backing_dev_info, BDI_RECLAIMABLE);
  389. __mark_inode_dirty(inode, I_DIRTY_DATASYNC);
  390. }
  391. static int
  392. nfs_clear_request_commit(struct nfs_page *req)
  393. {
  394. struct page *page = req->wb_page;
  395. if (test_and_clear_bit(PG_CLEAN, &(req)->wb_flags)) {
  396. dec_zone_page_state(page, NR_UNSTABLE_NFS);
  397. dec_bdi_stat(page->mapping->backing_dev_info, BDI_RECLAIMABLE);
  398. return 1;
  399. }
  400. return 0;
  401. }
  402. static inline
  403. int nfs_write_need_commit(struct nfs_write_data *data)
  404. {
  405. if (data->verf.committed == NFS_DATA_SYNC)
  406. return data->lseg == NULL;
  407. else
  408. return data->verf.committed != NFS_FILE_SYNC;
  409. }
  410. static inline
  411. int nfs_reschedule_unstable_write(struct nfs_page *req,
  412. struct nfs_write_data *data)
  413. {
  414. if (test_and_clear_bit(PG_NEED_COMMIT, &req->wb_flags)) {
  415. nfs_mark_request_commit(req, data->lseg);
  416. return 1;
  417. }
  418. if (test_and_clear_bit(PG_NEED_RESCHED, &req->wb_flags)) {
  419. nfs_mark_request_dirty(req);
  420. return 1;
  421. }
  422. return 0;
  423. }
  424. #else
  425. static inline void
  426. nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg)
  427. {
  428. }
  429. static inline int
  430. nfs_clear_request_commit(struct nfs_page *req)
  431. {
  432. return 0;
  433. }
  434. static inline
  435. int nfs_write_need_commit(struct nfs_write_data *data)
  436. {
  437. return 0;
  438. }
  439. static inline
  440. int nfs_reschedule_unstable_write(struct nfs_page *req,
  441. struct nfs_write_data *data)
  442. {
  443. return 0;
  444. }
  445. #endif
  446. #if defined(CONFIG_NFS_V3) || defined(CONFIG_NFS_V4)
  447. static int
  448. nfs_need_commit(struct nfs_inode *nfsi)
  449. {
  450. return radix_tree_tagged(&nfsi->nfs_page_tree, NFS_PAGE_TAG_COMMIT);
  451. }
  452. /*
  453. * nfs_scan_commit - Scan an inode for commit requests
  454. * @inode: NFS inode to scan
  455. * @dst: destination list
  456. * @idx_start: lower bound of page->index to scan.
  457. * @npages: idx_start + npages sets the upper bound to scan.
  458. *
  459. * Moves requests from the inode's 'commit' request list.
  460. * The requests are *not* checked to ensure that they form a contiguous set.
  461. */
  462. static int
  463. nfs_scan_commit(struct inode *inode, struct list_head *dst, pgoff_t idx_start, unsigned int npages)
  464. {
  465. struct nfs_inode *nfsi = NFS_I(inode);
  466. int ret;
  467. if (!nfs_need_commit(nfsi))
  468. return 0;
  469. spin_lock(&inode->i_lock);
  470. ret = nfs_scan_list(nfsi, dst, idx_start, npages, NFS_PAGE_TAG_COMMIT);
  471. if (ret > 0)
  472. nfsi->ncommit -= ret;
  473. spin_unlock(&inode->i_lock);
  474. if (nfs_need_commit(NFS_I(inode)))
  475. __mark_inode_dirty(inode, I_DIRTY_DATASYNC);
  476. return ret;
  477. }
  478. #else
  479. static inline int nfs_need_commit(struct nfs_inode *nfsi)
  480. {
  481. return 0;
  482. }
  483. static inline int nfs_scan_commit(struct inode *inode, struct list_head *dst, pgoff_t idx_start, unsigned int npages)
  484. {
  485. return 0;
  486. }
  487. #endif
  488. /*
  489. * Search for an existing write request, and attempt to update
  490. * it to reflect a new dirty region on a given page.
  491. *
  492. * If the attempt fails, then the existing request is flushed out
  493. * to disk.
  494. */
  495. static struct nfs_page *nfs_try_to_update_request(struct inode *inode,
  496. struct page *page,
  497. unsigned int offset,
  498. unsigned int bytes)
  499. {
  500. struct nfs_page *req;
  501. unsigned int rqend;
  502. unsigned int end;
  503. int error;
  504. if (!PagePrivate(page))
  505. return NULL;
  506. end = offset + bytes;
  507. spin_lock(&inode->i_lock);
  508. for (;;) {
  509. req = nfs_page_find_request_locked(page);
  510. if (req == NULL)
  511. goto out_unlock;
  512. rqend = req->wb_offset + req->wb_bytes;
  513. /*
  514. * Tell the caller to flush out the request if
  515. * the offsets are non-contiguous.
  516. * Note: nfs_flush_incompatible() will already
  517. * have flushed out requests having wrong owners.
  518. */
  519. if (offset > rqend
  520. || end < req->wb_offset)
  521. goto out_flushme;
  522. if (nfs_set_page_tag_locked(req))
  523. break;
  524. /* The request is locked, so wait and then retry */
  525. spin_unlock(&inode->i_lock);
  526. error = nfs_wait_on_request(req);
  527. nfs_release_request(req);
  528. if (error != 0)
  529. goto out_err;
  530. spin_lock(&inode->i_lock);
  531. }
  532. if (nfs_clear_request_commit(req) &&
  533. radix_tree_tag_clear(&NFS_I(inode)->nfs_page_tree,
  534. req->wb_index, NFS_PAGE_TAG_COMMIT) != NULL) {
  535. NFS_I(inode)->ncommit--;
  536. pnfs_clear_request_commit(req);
  537. }
  538. /* Okay, the request matches. Update the region */
  539. if (offset < req->wb_offset) {
  540. req->wb_offset = offset;
  541. req->wb_pgbase = offset;
  542. }
  543. if (end > rqend)
  544. req->wb_bytes = end - req->wb_offset;
  545. else
  546. req->wb_bytes = rqend - req->wb_offset;
  547. out_unlock:
  548. spin_unlock(&inode->i_lock);
  549. return req;
  550. out_flushme:
  551. spin_unlock(&inode->i_lock);
  552. nfs_release_request(req);
  553. error = nfs_wb_page(inode, page);
  554. out_err:
  555. return ERR_PTR(error);
  556. }
  557. /*
  558. * Try to update an existing write request, or create one if there is none.
  559. *
  560. * Note: Should always be called with the Page Lock held to prevent races
  561. * if we have to add a new request. Also assumes that the caller has
  562. * already called nfs_flush_incompatible() if necessary.
  563. */
  564. static struct nfs_page * nfs_setup_write_request(struct nfs_open_context* ctx,
  565. struct page *page, unsigned int offset, unsigned int bytes)
  566. {
  567. struct inode *inode = page->mapping->host;
  568. struct nfs_page *req;
  569. int error;
  570. req = nfs_try_to_update_request(inode, page, offset, bytes);
  571. if (req != NULL)
  572. goto out;
  573. req = nfs_create_request(ctx, inode, page, offset, bytes);
  574. if (IS_ERR(req))
  575. goto out;
  576. error = nfs_inode_add_request(inode, req);
  577. if (error != 0) {
  578. nfs_release_request(req);
  579. req = ERR_PTR(error);
  580. }
  581. out:
  582. return req;
  583. }
  584. static int nfs_writepage_setup(struct nfs_open_context *ctx, struct page *page,
  585. unsigned int offset, unsigned int count)
  586. {
  587. struct nfs_page *req;
  588. req = nfs_setup_write_request(ctx, page, offset, count);
  589. if (IS_ERR(req))
  590. return PTR_ERR(req);
  591. /* Update file length */
  592. nfs_grow_file(page, offset, count);
  593. nfs_mark_uptodate(page, req->wb_pgbase, req->wb_bytes);
  594. nfs_mark_request_dirty(req);
  595. nfs_clear_page_tag_locked(req);
  596. return 0;
  597. }
  598. int nfs_flush_incompatible(struct file *file, struct page *page)
  599. {
  600. struct nfs_open_context *ctx = nfs_file_open_context(file);
  601. struct nfs_page *req;
  602. int do_flush, status;
  603. /*
  604. * Look for a request corresponding to this page. If there
  605. * is one, and it belongs to another file, we flush it out
  606. * before we try to copy anything into the page. Do this
  607. * due to the lack of an ACCESS-type call in NFSv2.
  608. * Also do the same if we find a request from an existing
  609. * dropped page.
  610. */
  611. do {
  612. req = nfs_page_find_request(page);
  613. if (req == NULL)
  614. return 0;
  615. do_flush = req->wb_page != page || req->wb_context != ctx ||
  616. req->wb_lock_context->lockowner != current->files ||
  617. req->wb_lock_context->pid != current->tgid;
  618. nfs_release_request(req);
  619. if (!do_flush)
  620. return 0;
  621. status = nfs_wb_page(page->mapping->host, page);
  622. } while (status == 0);
  623. return status;
  624. }
  625. /*
  626. * If the page cache is marked as unsafe or invalid, then we can't rely on
  627. * the PageUptodate() flag. In this case, we will need to turn off
  628. * write optimisations that depend on the page contents being correct.
  629. */
  630. static int nfs_write_pageuptodate(struct page *page, struct inode *inode)
  631. {
  632. return PageUptodate(page) &&
  633. !(NFS_I(inode)->cache_validity & (NFS_INO_REVAL_PAGECACHE|NFS_INO_INVALID_DATA));
  634. }
  635. /*
  636. * Update and possibly write a cached page of an NFS file.
  637. *
  638. * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad
  639. * things with a page scheduled for an RPC call (e.g. invalidate it).
  640. */
  641. int nfs_updatepage(struct file *file, struct page *page,
  642. unsigned int offset, unsigned int count)
  643. {
  644. struct nfs_open_context *ctx = nfs_file_open_context(file);
  645. struct inode *inode = page->mapping->host;
  646. int status = 0;
  647. nfs_inc_stats(inode, NFSIOS_VFSUPDATEPAGE);
  648. dprintk("NFS: nfs_updatepage(%s/%s %d@%lld)\n",
  649. file->f_path.dentry->d_parent->d_name.name,
  650. file->f_path.dentry->d_name.name, count,
  651. (long long)(page_offset(page) + offset));
  652. /* If we're not using byte range locks, and we know the page
  653. * is up to date, it may be more efficient to extend the write
  654. * to cover the entire page in order to avoid fragmentation
  655. * inefficiencies.
  656. */
  657. if (nfs_write_pageuptodate(page, inode) &&
  658. inode->i_flock == NULL &&
  659. !(file->f_flags & O_DSYNC)) {
  660. count = max(count + offset, nfs_page_length(page));
  661. offset = 0;
  662. }
  663. status = nfs_writepage_setup(ctx, page, offset, count);
  664. if (status < 0)
  665. nfs_set_pageerror(page);
  666. else
  667. __set_page_dirty_nobuffers(page);
  668. dprintk("NFS: nfs_updatepage returns %d (isize %lld)\n",
  669. status, (long long)i_size_read(inode));
  670. return status;
  671. }
  672. static void nfs_writepage_release(struct nfs_page *req,
  673. struct nfs_write_data *data)
  674. {
  675. struct page *page = req->wb_page;
  676. if (PageError(req->wb_page) || !nfs_reschedule_unstable_write(req, data))
  677. nfs_inode_remove_request(req);
  678. nfs_clear_page_tag_locked(req);
  679. nfs_end_page_writeback(page);
  680. }
  681. static int flush_task_priority(int how)
  682. {
  683. switch (how & (FLUSH_HIGHPRI|FLUSH_LOWPRI)) {
  684. case FLUSH_HIGHPRI:
  685. return RPC_PRIORITY_HIGH;
  686. case FLUSH_LOWPRI:
  687. return RPC_PRIORITY_LOW;
  688. }
  689. return RPC_PRIORITY_NORMAL;
  690. }
  691. int nfs_initiate_write(struct nfs_write_data *data,
  692. struct rpc_clnt *clnt,
  693. const struct rpc_call_ops *call_ops,
  694. int how)
  695. {
  696. struct inode *inode = data->inode;
  697. int priority = flush_task_priority(how);
  698. struct rpc_task *task;
  699. struct rpc_message msg = {
  700. .rpc_argp = &data->args,
  701. .rpc_resp = &data->res,
  702. .rpc_cred = data->cred,
  703. };
  704. struct rpc_task_setup task_setup_data = {
  705. .rpc_client = clnt,
  706. .task = &data->task,
  707. .rpc_message = &msg,
  708. .callback_ops = call_ops,
  709. .callback_data = data,
  710. .workqueue = nfsiod_workqueue,
  711. .flags = RPC_TASK_ASYNC,
  712. .priority = priority,
  713. };
  714. int ret = 0;
  715. /* Set up the initial task struct. */
  716. NFS_PROTO(inode)->write_setup(data, &msg);
  717. dprintk("NFS: %5u initiated write call "
  718. "(req %s/%lld, %u bytes @ offset %llu)\n",
  719. data->task.tk_pid,
  720. inode->i_sb->s_id,
  721. (long long)NFS_FILEID(inode),
  722. data->args.count,
  723. (unsigned long long)data->args.offset);
  724. task = rpc_run_task(&task_setup_data);
  725. if (IS_ERR(task)) {
  726. ret = PTR_ERR(task);
  727. goto out;
  728. }
  729. if (how & FLUSH_SYNC) {
  730. ret = rpc_wait_for_completion_task(task);
  731. if (ret == 0)
  732. ret = task->tk_status;
  733. }
  734. rpc_put_task(task);
  735. out:
  736. return ret;
  737. }
  738. EXPORT_SYMBOL_GPL(nfs_initiate_write);
  739. /*
  740. * Set up the argument/result storage required for the RPC call.
  741. */
  742. static void nfs_write_rpcsetup(struct nfs_page *req,
  743. struct nfs_write_data *data,
  744. unsigned int count, unsigned int offset,
  745. int how)
  746. {
  747. struct inode *inode = req->wb_context->dentry->d_inode;
  748. /* Set up the RPC argument and reply structs
  749. * NB: take care not to mess about with data->commit et al. */
  750. data->req = req;
  751. data->inode = inode = req->wb_context->dentry->d_inode;
  752. data->cred = req->wb_context->cred;
  753. data->args.fh = NFS_FH(inode);
  754. data->args.offset = req_offset(req) + offset;
  755. /* pnfs_set_layoutcommit needs this */
  756. data->mds_offset = data->args.offset;
  757. data->args.pgbase = req->wb_pgbase + offset;
  758. data->args.pages = data->pagevec;
  759. data->args.count = count;
  760. data->args.context = get_nfs_open_context(req->wb_context);
  761. data->args.lock_context = req->wb_lock_context;
  762. data->args.stable = NFS_UNSTABLE;
  763. switch (how & (FLUSH_STABLE | FLUSH_COND_STABLE)) {
  764. case 0:
  765. break;
  766. case FLUSH_COND_STABLE:
  767. if (nfs_need_commit(NFS_I(inode)))
  768. break;
  769. default:
  770. data->args.stable = NFS_FILE_SYNC;
  771. }
  772. data->res.fattr = &data->fattr;
  773. data->res.count = count;
  774. data->res.verf = &data->verf;
  775. nfs_fattr_init(&data->fattr);
  776. }
  777. static int nfs_do_write(struct nfs_write_data *data,
  778. const struct rpc_call_ops *call_ops,
  779. int how)
  780. {
  781. struct inode *inode = data->args.context->dentry->d_inode;
  782. return nfs_initiate_write(data, NFS_CLIENT(inode), call_ops, how);
  783. }
  784. static int nfs_do_multiple_writes(struct list_head *head,
  785. const struct rpc_call_ops *call_ops,
  786. int how)
  787. {
  788. struct nfs_write_data *data;
  789. int ret = 0;
  790. while (!list_empty(head)) {
  791. int ret2;
  792. data = list_entry(head->next, struct nfs_write_data, list);
  793. list_del_init(&data->list);
  794. ret2 = nfs_do_write(data, call_ops, how);
  795. if (ret == 0)
  796. ret = ret2;
  797. }
  798. return ret;
  799. }
  800. /* If a nfs_flush_* function fails, it should remove reqs from @head and
  801. * call this on each, which will prepare them to be retried on next
  802. * writeback using standard nfs.
  803. */
  804. static void nfs_redirty_request(struct nfs_page *req)
  805. {
  806. struct page *page = req->wb_page;
  807. nfs_mark_request_dirty(req);
  808. nfs_clear_page_tag_locked(req);
  809. nfs_end_page_writeback(page);
  810. }
  811. /*
  812. * Generate multiple small requests to write out a single
  813. * contiguous dirty area on one page.
  814. */
  815. static int nfs_flush_multi(struct nfs_pageio_descriptor *desc, struct list_head *res)
  816. {
  817. struct nfs_page *req = nfs_list_entry(desc->pg_list.next);
  818. struct page *page = req->wb_page;
  819. struct nfs_write_data *data;
  820. size_t wsize = desc->pg_bsize, nbytes;
  821. unsigned int offset;
  822. int requests = 0;
  823. int ret = 0;
  824. nfs_list_remove_request(req);
  825. if ((desc->pg_ioflags & FLUSH_COND_STABLE) &&
  826. (desc->pg_moreio || NFS_I(desc->pg_inode)->ncommit ||
  827. desc->pg_count > wsize))
  828. desc->pg_ioflags &= ~FLUSH_COND_STABLE;
  829. offset = 0;
  830. nbytes = desc->pg_count;
  831. do {
  832. size_t len = min(nbytes, wsize);
  833. data = nfs_writedata_alloc(1);
  834. if (!data)
  835. goto out_bad;
  836. data->pagevec[0] = page;
  837. nfs_write_rpcsetup(req, data, len, offset, desc->pg_ioflags);
  838. list_add(&data->list, res);
  839. requests++;
  840. nbytes -= len;
  841. offset += len;
  842. } while (nbytes != 0);
  843. atomic_set(&req->wb_complete, requests);
  844. desc->pg_rpc_callops = &nfs_write_partial_ops;
  845. return ret;
  846. out_bad:
  847. while (!list_empty(res)) {
  848. data = list_entry(res->next, struct nfs_write_data, list);
  849. list_del(&data->list);
  850. nfs_writedata_free(data);
  851. }
  852. nfs_redirty_request(req);
  853. return -ENOMEM;
  854. }
  855. /*
  856. * Create an RPC task for the given write request and kick it.
  857. * The page must have been locked by the caller.
  858. *
  859. * It may happen that the page we're passed is not marked dirty.
  860. * This is the case if nfs_updatepage detects a conflicting request
  861. * that has been written but not committed.
  862. */
  863. static int nfs_flush_one(struct nfs_pageio_descriptor *desc, struct list_head *res)
  864. {
  865. struct nfs_page *req;
  866. struct page **pages;
  867. struct nfs_write_data *data;
  868. struct list_head *head = &desc->pg_list;
  869. int ret = 0;
  870. data = nfs_writedata_alloc(nfs_page_array_len(desc->pg_base,
  871. desc->pg_count));
  872. if (!data) {
  873. while (!list_empty(head)) {
  874. req = nfs_list_entry(head->next);
  875. nfs_list_remove_request(req);
  876. nfs_redirty_request(req);
  877. }
  878. ret = -ENOMEM;
  879. goto out;
  880. }
  881. pages = data->pagevec;
  882. while (!list_empty(head)) {
  883. req = nfs_list_entry(head->next);
  884. nfs_list_remove_request(req);
  885. nfs_list_add_request(req, &data->pages);
  886. *pages++ = req->wb_page;
  887. }
  888. req = nfs_list_entry(data->pages.next);
  889. if ((desc->pg_ioflags & FLUSH_COND_STABLE) &&
  890. (desc->pg_moreio || NFS_I(desc->pg_inode)->ncommit))
  891. desc->pg_ioflags &= ~FLUSH_COND_STABLE;
  892. /* Set up the argument struct */
  893. nfs_write_rpcsetup(req, data, desc->pg_count, 0, desc->pg_ioflags);
  894. list_add(&data->list, res);
  895. desc->pg_rpc_callops = &nfs_write_full_ops;
  896. out:
  897. return ret;
  898. }
  899. int nfs_generic_flush(struct nfs_pageio_descriptor *desc, struct list_head *head)
  900. {
  901. if (desc->pg_bsize < PAGE_CACHE_SIZE)
  902. return nfs_flush_multi(desc, head);
  903. return nfs_flush_one(desc, head);
  904. }
  905. static int nfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc)
  906. {
  907. LIST_HEAD(head);
  908. int ret;
  909. ret = nfs_generic_flush(desc, &head);
  910. if (ret == 0)
  911. ret = nfs_do_multiple_writes(&head, desc->pg_rpc_callops,
  912. desc->pg_ioflags);
  913. return ret;
  914. }
  915. static const struct nfs_pageio_ops nfs_pageio_write_ops = {
  916. .pg_test = nfs_generic_pg_test,
  917. .pg_doio = nfs_generic_pg_writepages,
  918. };
  919. static void nfs_pageio_init_write_mds(struct nfs_pageio_descriptor *pgio,
  920. struct inode *inode, int ioflags)
  921. {
  922. nfs_pageio_init(pgio, inode, &nfs_pageio_write_ops,
  923. NFS_SERVER(inode)->wsize, ioflags);
  924. }
  925. void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio)
  926. {
  927. pgio->pg_ops = &nfs_pageio_write_ops;
  928. pgio->pg_bsize = NFS_SERVER(pgio->pg_inode)->wsize;
  929. }
  930. EXPORT_SYMBOL_GPL(nfs_pageio_reset_write_mds);
  931. static void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio,
  932. struct inode *inode, int ioflags)
  933. {
  934. if (!pnfs_pageio_init_write(pgio, inode, ioflags))
  935. nfs_pageio_init_write_mds(pgio, inode, ioflags);
  936. }
  937. /*
  938. * Handle a write reply that flushed part of a page.
  939. */
  940. static void nfs_writeback_done_partial(struct rpc_task *task, void *calldata)
  941. {
  942. struct nfs_write_data *data = calldata;
  943. dprintk("NFS: %5u write(%s/%lld %d@%lld)",
  944. task->tk_pid,
  945. data->req->wb_context->dentry->d_inode->i_sb->s_id,
  946. (long long)
  947. NFS_FILEID(data->req->wb_context->dentry->d_inode),
  948. data->req->wb_bytes, (long long)req_offset(data->req));
  949. nfs_writeback_done(task, data);
  950. }
  951. static void nfs_writeback_release_partial(void *calldata)
  952. {
  953. struct nfs_write_data *data = calldata;
  954. struct nfs_page *req = data->req;
  955. struct page *page = req->wb_page;
  956. int status = data->task.tk_status;
  957. if (status < 0) {
  958. nfs_set_pageerror(page);
  959. nfs_context_set_write_error(req->wb_context, status);
  960. dprintk(", error = %d\n", status);
  961. goto out;
  962. }
  963. if (nfs_write_need_commit(data)) {
  964. struct inode *inode = page->mapping->host;
  965. spin_lock(&inode->i_lock);
  966. if (test_bit(PG_NEED_RESCHED, &req->wb_flags)) {
  967. /* Do nothing we need to resend the writes */
  968. } else if (!test_and_set_bit(PG_NEED_COMMIT, &req->wb_flags)) {
  969. memcpy(&req->wb_verf, &data->verf, sizeof(req->wb_verf));
  970. dprintk(" defer commit\n");
  971. } else if (memcmp(&req->wb_verf, &data->verf, sizeof(req->wb_verf))) {
  972. set_bit(PG_NEED_RESCHED, &req->wb_flags);
  973. clear_bit(PG_NEED_COMMIT, &req->wb_flags);
  974. dprintk(" server reboot detected\n");
  975. }
  976. spin_unlock(&inode->i_lock);
  977. } else
  978. dprintk(" OK\n");
  979. out:
  980. if (atomic_dec_and_test(&req->wb_complete))
  981. nfs_writepage_release(req, data);
  982. nfs_writedata_release(calldata);
  983. }
  984. #if defined(CONFIG_NFS_V4_1)
  985. void nfs_write_prepare(struct rpc_task *task, void *calldata)
  986. {
  987. struct nfs_write_data *data = calldata;
  988. if (nfs4_setup_sequence(NFS_SERVER(data->inode),
  989. &data->args.seq_args,
  990. &data->res.seq_res, 1, task))
  991. return;
  992. rpc_call_start(task);
  993. }
  994. #endif /* CONFIG_NFS_V4_1 */
  995. static const struct rpc_call_ops nfs_write_partial_ops = {
  996. #if defined(CONFIG_NFS_V4_1)
  997. .rpc_call_prepare = nfs_write_prepare,
  998. #endif /* CONFIG_NFS_V4_1 */
  999. .rpc_call_done = nfs_writeback_done_partial,
  1000. .rpc_release = nfs_writeback_release_partial,
  1001. };
  1002. /*
  1003. * Handle a write reply that flushes a whole page.
  1004. *
  1005. * FIXME: There is an inherent race with invalidate_inode_pages and
  1006. * writebacks since the page->count is kept > 1 for as long
  1007. * as the page has a write request pending.
  1008. */
  1009. static void nfs_writeback_done_full(struct rpc_task *task, void *calldata)
  1010. {
  1011. struct nfs_write_data *data = calldata;
  1012. nfs_writeback_done(task, data);
  1013. }
  1014. static void nfs_writeback_release_full(void *calldata)
  1015. {
  1016. struct nfs_write_data *data = calldata;
  1017. int ret, status = data->task.tk_status;
  1018. struct nfs_pageio_descriptor pgio;
  1019. if (data->pnfs_error) {
  1020. nfs_pageio_init_write_mds(&pgio, data->inode, FLUSH_STABLE);
  1021. pgio.pg_recoalesce = 1;
  1022. }
  1023. /* Update attributes as result of writeback. */
  1024. while (!list_empty(&data->pages)) {
  1025. struct nfs_page *req = nfs_list_entry(data->pages.next);
  1026. struct page *page = req->wb_page;
  1027. nfs_list_remove_request(req);
  1028. dprintk("NFS: %5u write (%s/%lld %d@%lld)",
  1029. data->task.tk_pid,
  1030. req->wb_context->dentry->d_inode->i_sb->s_id,
  1031. (long long)NFS_FILEID(req->wb_context->dentry->d_inode),
  1032. req->wb_bytes,
  1033. (long long)req_offset(req));
  1034. if (data->pnfs_error) {
  1035. dprintk(", pnfs error = %d\n", data->pnfs_error);
  1036. goto next;
  1037. }
  1038. if (status < 0) {
  1039. nfs_set_pageerror(page);
  1040. nfs_context_set_write_error(req->wb_context, status);
  1041. dprintk(", error = %d\n", status);
  1042. goto remove_request;
  1043. }
  1044. if (nfs_write_need_commit(data)) {
  1045. memcpy(&req->wb_verf, &data->verf, sizeof(req->wb_verf));
  1046. nfs_mark_request_commit(req, data->lseg);
  1047. dprintk(" marked for commit\n");
  1048. goto next;
  1049. }
  1050. dprintk(" OK\n");
  1051. remove_request:
  1052. nfs_inode_remove_request(req);
  1053. next:
  1054. nfs_clear_page_tag_locked(req);
  1055. nfs_end_page_writeback(page);
  1056. if (data->pnfs_error) {
  1057. lock_page(page);
  1058. nfs_pageio_cond_complete(&pgio, page->index);
  1059. ret = nfs_page_async_flush(&pgio, page, 0);
  1060. if (ret) {
  1061. nfs_set_pageerror(page);
  1062. dprintk("rewrite to MDS error = %d\n", ret);
  1063. }
  1064. unlock_page(page);
  1065. }
  1066. }
  1067. if (data->pnfs_error)
  1068. nfs_pageio_complete(&pgio);
  1069. nfs_writedata_release(calldata);
  1070. }
  1071. static const struct rpc_call_ops nfs_write_full_ops = {
  1072. #if defined(CONFIG_NFS_V4_1)
  1073. .rpc_call_prepare = nfs_write_prepare,
  1074. #endif /* CONFIG_NFS_V4_1 */
  1075. .rpc_call_done = nfs_writeback_done_full,
  1076. .rpc_release = nfs_writeback_release_full,
  1077. };
  1078. /*
  1079. * This function is called when the WRITE call is complete.
  1080. */
  1081. void nfs_writeback_done(struct rpc_task *task, struct nfs_write_data *data)
  1082. {
  1083. struct nfs_writeargs *argp = &data->args;
  1084. struct nfs_writeres *resp = &data->res;
  1085. int status;
  1086. dprintk("NFS: %5u nfs_writeback_done (status %d)\n",
  1087. task->tk_pid, task->tk_status);
  1088. /*
  1089. * ->write_done will attempt to use post-op attributes to detect
  1090. * conflicting writes by other clients. A strict interpretation
  1091. * of close-to-open would allow us to continue caching even if
  1092. * another writer had changed the file, but some applications
  1093. * depend on tighter cache coherency when writing.
  1094. */
  1095. status = NFS_PROTO(data->inode)->write_done(task, data);
  1096. if (status != 0)
  1097. return;
  1098. nfs_add_stats(data->inode, NFSIOS_SERVERWRITTENBYTES, resp->count);
  1099. #if defined(CONFIG_NFS_V3) || defined(CONFIG_NFS_V4)
  1100. if (resp->verf->committed < argp->stable && task->tk_status >= 0) {
  1101. /* We tried a write call, but the server did not
  1102. * commit data to stable storage even though we
  1103. * requested it.
  1104. * Note: There is a known bug in Tru64 < 5.0 in which
  1105. * the server reports NFS_DATA_SYNC, but performs
  1106. * NFS_FILE_SYNC. We therefore implement this checking
  1107. * as a dprintk() in order to avoid filling syslog.
  1108. */
  1109. static unsigned long complain;
  1110. /* Note this will print the MDS for a DS write */
  1111. if (time_before(complain, jiffies)) {
  1112. dprintk("NFS: faulty NFS server %s:"
  1113. " (committed = %d) != (stable = %d)\n",
  1114. NFS_SERVER(data->inode)->nfs_client->cl_hostname,
  1115. resp->verf->committed, argp->stable);
  1116. complain = jiffies + 300 * HZ;
  1117. }
  1118. }
  1119. #endif
  1120. /* Is this a short write? */
  1121. if (task->tk_status >= 0 && resp->count < argp->count) {
  1122. static unsigned long complain;
  1123. nfs_inc_stats(data->inode, NFSIOS_SHORTWRITE);
  1124. /* Has the server at least made some progress? */
  1125. if (resp->count != 0) {
  1126. /* Was this an NFSv2 write or an NFSv3 stable write? */
  1127. if (resp->verf->committed != NFS_UNSTABLE) {
  1128. /* Resend from where the server left off */
  1129. data->mds_offset += resp->count;
  1130. argp->offset += resp->count;
  1131. argp->pgbase += resp->count;
  1132. argp->count -= resp->count;
  1133. } else {
  1134. /* Resend as a stable write in order to avoid
  1135. * headaches in the case of a server crash.
  1136. */
  1137. argp->stable = NFS_FILE_SYNC;
  1138. }
  1139. rpc_restart_call_prepare(task);
  1140. return;
  1141. }
  1142. if (time_before(complain, jiffies)) {
  1143. printk(KERN_WARNING
  1144. "NFS: Server wrote zero bytes, expected %u.\n",
  1145. argp->count);
  1146. complain = jiffies + 300 * HZ;
  1147. }
  1148. /* Can't do anything about it except throw an error. */
  1149. task->tk_status = -EIO;
  1150. }
  1151. return;
  1152. }
  1153. #if defined(CONFIG_NFS_V3) || defined(CONFIG_NFS_V4)
  1154. static int nfs_commit_set_lock(struct nfs_inode *nfsi, int may_wait)
  1155. {
  1156. int ret;
  1157. if (!test_and_set_bit(NFS_INO_COMMIT, &nfsi->flags))
  1158. return 1;
  1159. if (!may_wait)
  1160. return 0;
  1161. ret = out_of_line_wait_on_bit_lock(&nfsi->flags,
  1162. NFS_INO_COMMIT,
  1163. nfs_wait_bit_killable,
  1164. TASK_KILLABLE);
  1165. return (ret < 0) ? ret : 1;
  1166. }
  1167. void nfs_commit_clear_lock(struct nfs_inode *nfsi)
  1168. {
  1169. clear_bit(NFS_INO_COMMIT, &nfsi->flags);
  1170. smp_mb__after_clear_bit();
  1171. wake_up_bit(&nfsi->flags, NFS_INO_COMMIT);
  1172. }
  1173. EXPORT_SYMBOL_GPL(nfs_commit_clear_lock);
  1174. void nfs_commitdata_release(void *data)
  1175. {
  1176. struct nfs_write_data *wdata = data;
  1177. put_lseg(wdata->lseg);
  1178. put_nfs_open_context(wdata->args.context);
  1179. nfs_commit_free(wdata);
  1180. }
  1181. EXPORT_SYMBOL_GPL(nfs_commitdata_release);
  1182. int nfs_initiate_commit(struct nfs_write_data *data, struct rpc_clnt *clnt,
  1183. const struct rpc_call_ops *call_ops,
  1184. int how)
  1185. {
  1186. struct rpc_task *task;
  1187. int priority = flush_task_priority(how);
  1188. struct rpc_message msg = {
  1189. .rpc_argp = &data->args,
  1190. .rpc_resp = &data->res,
  1191. .rpc_cred = data->cred,
  1192. };
  1193. struct rpc_task_setup task_setup_data = {
  1194. .task = &data->task,
  1195. .rpc_client = clnt,
  1196. .rpc_message = &msg,
  1197. .callback_ops = call_ops,
  1198. .callback_data = data,
  1199. .workqueue = nfsiod_workqueue,
  1200. .flags = RPC_TASK_ASYNC,
  1201. .priority = priority,
  1202. };
  1203. /* Set up the initial task struct. */
  1204. NFS_PROTO(data->inode)->commit_setup(data, &msg);
  1205. dprintk("NFS: %5u initiated commit call\n", data->task.tk_pid);
  1206. task = rpc_run_task(&task_setup_data);
  1207. if (IS_ERR(task))
  1208. return PTR_ERR(task);
  1209. if (how & FLUSH_SYNC)
  1210. rpc_wait_for_completion_task(task);
  1211. rpc_put_task(task);
  1212. return 0;
  1213. }
  1214. EXPORT_SYMBOL_GPL(nfs_initiate_commit);
  1215. /*
  1216. * Set up the argument/result storage required for the RPC call.
  1217. */
  1218. void nfs_init_commit(struct nfs_write_data *data,
  1219. struct list_head *head,
  1220. struct pnfs_layout_segment *lseg)
  1221. {
  1222. struct nfs_page *first = nfs_list_entry(head->next);
  1223. struct inode *inode = first->wb_context->dentry->d_inode;
  1224. /* Set up the RPC argument and reply structs
  1225. * NB: take care not to mess about with data->commit et al. */
  1226. list_splice_init(head, &data->pages);
  1227. data->inode = inode;
  1228. data->cred = first->wb_context->cred;
  1229. data->lseg = lseg; /* reference transferred */
  1230. data->mds_ops = &nfs_commit_ops;
  1231. data->args.fh = NFS_FH(data->inode);
  1232. /* Note: we always request a commit of the entire inode */
  1233. data->args.offset = 0;
  1234. data->args.count = 0;
  1235. data->args.context = get_nfs_open_context(first->wb_context);
  1236. data->res.count = 0;
  1237. data->res.fattr = &data->fattr;
  1238. data->res.verf = &data->verf;
  1239. nfs_fattr_init(&data->fattr);
  1240. }
  1241. EXPORT_SYMBOL_GPL(nfs_init_commit);
  1242. void nfs_retry_commit(struct list_head *page_list,
  1243. struct pnfs_layout_segment *lseg)
  1244. {
  1245. struct nfs_page *req;
  1246. while (!list_empty(page_list)) {
  1247. req = nfs_list_entry(page_list->next);
  1248. nfs_list_remove_request(req);
  1249. nfs_mark_request_commit(req, lseg);
  1250. dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
  1251. dec_bdi_stat(req->wb_page->mapping->backing_dev_info,
  1252. BDI_RECLAIMABLE);
  1253. nfs_clear_page_tag_locked(req);
  1254. }
  1255. }
  1256. EXPORT_SYMBOL_GPL(nfs_retry_commit);
  1257. /*
  1258. * Commit dirty pages
  1259. */
  1260. static int
  1261. nfs_commit_list(struct inode *inode, struct list_head *head, int how)
  1262. {
  1263. struct nfs_write_data *data;
  1264. data = nfs_commitdata_alloc();
  1265. if (!data)
  1266. goto out_bad;
  1267. /* Set up the argument struct */
  1268. nfs_init_commit(data, head, NULL);
  1269. return nfs_initiate_commit(data, NFS_CLIENT(inode), data->mds_ops, how);
  1270. out_bad:
  1271. nfs_retry_commit(head, NULL);
  1272. nfs_commit_clear_lock(NFS_I(inode));
  1273. return -ENOMEM;
  1274. }
  1275. /*
  1276. * COMMIT call returned
  1277. */
  1278. static void nfs_commit_done(struct rpc_task *task, void *calldata)
  1279. {
  1280. struct nfs_write_data *data = calldata;
  1281. dprintk("NFS: %5u nfs_commit_done (status %d)\n",
  1282. task->tk_pid, task->tk_status);
  1283. /* Call the NFS version-specific code */
  1284. NFS_PROTO(data->inode)->commit_done(task, data);
  1285. }
  1286. void nfs_commit_release_pages(struct nfs_write_data *data)
  1287. {
  1288. struct nfs_page *req;
  1289. int status = data->task.tk_status;
  1290. while (!list_empty(&data->pages)) {
  1291. req = nfs_list_entry(data->pages.next);
  1292. nfs_list_remove_request(req);
  1293. nfs_clear_request_commit(req);
  1294. dprintk("NFS: commit (%s/%lld %d@%lld)",
  1295. req->wb_context->dentry->d_sb->s_id,
  1296. (long long)NFS_FILEID(req->wb_context->dentry->d_inode),
  1297. req->wb_bytes,
  1298. (long long)req_offset(req));
  1299. if (status < 0) {
  1300. nfs_context_set_write_error(req->wb_context, status);
  1301. nfs_inode_remove_request(req);
  1302. dprintk(", error = %d\n", status);
  1303. goto next;
  1304. }
  1305. /* Okay, COMMIT succeeded, apparently. Check the verifier
  1306. * returned by the server against all stored verfs. */
  1307. if (!memcmp(req->wb_verf.verifier, data->verf.verifier, sizeof(data->verf.verifier))) {
  1308. /* We have a match */
  1309. nfs_inode_remove_request(req);
  1310. dprintk(" OK\n");
  1311. goto next;
  1312. }
  1313. /* We have a mismatch. Write the page again */
  1314. dprintk(" mismatch\n");
  1315. nfs_mark_request_dirty(req);
  1316. next:
  1317. nfs_clear_page_tag_locked(req);
  1318. }
  1319. }
  1320. EXPORT_SYMBOL_GPL(nfs_commit_release_pages);
  1321. static void nfs_commit_release(void *calldata)
  1322. {
  1323. struct nfs_write_data *data = calldata;
  1324. nfs_commit_release_pages(data);
  1325. nfs_commit_clear_lock(NFS_I(data->inode));
  1326. nfs_commitdata_release(calldata);
  1327. }
  1328. static const struct rpc_call_ops nfs_commit_ops = {
  1329. #if defined(CONFIG_NFS_V4_1)
  1330. .rpc_call_prepare = nfs_write_prepare,
  1331. #endif /* CONFIG_NFS_V4_1 */
  1332. .rpc_call_done = nfs_commit_done,
  1333. .rpc_release = nfs_commit_release,
  1334. };
  1335. int nfs_commit_inode(struct inode *inode, int how)
  1336. {
  1337. LIST_HEAD(head);
  1338. int may_wait = how & FLUSH_SYNC;
  1339. int res;
  1340. res = nfs_commit_set_lock(NFS_I(inode), may_wait);
  1341. if (res <= 0)
  1342. goto out_mark_dirty;
  1343. res = nfs_scan_commit(inode, &head, 0, 0);
  1344. if (res) {
  1345. int error;
  1346. error = pnfs_commit_list(inode, &head, how);
  1347. if (error == PNFS_NOT_ATTEMPTED)
  1348. error = nfs_commit_list(inode, &head, how);
  1349. if (error < 0)
  1350. return error;
  1351. if (!may_wait)
  1352. goto out_mark_dirty;
  1353. error = wait_on_bit(&NFS_I(inode)->flags,
  1354. NFS_INO_COMMIT,
  1355. nfs_wait_bit_killable,
  1356. TASK_KILLABLE);
  1357. if (error < 0)
  1358. return error;
  1359. } else
  1360. nfs_commit_clear_lock(NFS_I(inode));
  1361. return res;
  1362. /* Note: If we exit without ensuring that the commit is complete,
  1363. * we must mark the inode as dirty. Otherwise, future calls to
  1364. * sync_inode() with the WB_SYNC_ALL flag set will fail to ensure
  1365. * that the data is on the disk.
  1366. */
  1367. out_mark_dirty:
  1368. __mark_inode_dirty(inode, I_DIRTY_DATASYNC);
  1369. return res;
  1370. }
  1371. static int nfs_commit_unstable_pages(struct inode *inode, struct writeback_control *wbc)
  1372. {
  1373. struct nfs_inode *nfsi = NFS_I(inode);
  1374. int flags = FLUSH_SYNC;
  1375. int ret = 0;
  1376. /* no commits means nothing needs to be done */
  1377. if (!nfsi->ncommit)
  1378. return ret;
  1379. if (wbc->sync_mode == WB_SYNC_NONE) {
  1380. /* Don't commit yet if this is a non-blocking flush and there
  1381. * are a lot of outstanding writes for this mapping.
  1382. */
  1383. if (nfsi->ncommit <= (nfsi->npages >> 1))
  1384. goto out_mark_dirty;
  1385. /* don't wait for the COMMIT response */
  1386. flags = 0;
  1387. }
  1388. ret = nfs_commit_inode(inode, flags);
  1389. if (ret >= 0) {
  1390. if (wbc->sync_mode == WB_SYNC_NONE) {
  1391. if (ret < wbc->nr_to_write)
  1392. wbc->nr_to_write -= ret;
  1393. else
  1394. wbc->nr_to_write = 0;
  1395. }
  1396. return 0;
  1397. }
  1398. out_mark_dirty:
  1399. __mark_inode_dirty(inode, I_DIRTY_DATASYNC);
  1400. return ret;
  1401. }
  1402. #else
  1403. static int nfs_commit_unstable_pages(struct inode *inode, struct writeback_control *wbc)
  1404. {
  1405. return 0;
  1406. }
  1407. #endif
  1408. int nfs_write_inode(struct inode *inode, struct writeback_control *wbc)
  1409. {
  1410. int ret;
  1411. ret = nfs_commit_unstable_pages(inode, wbc);
  1412. if (ret >= 0 && test_bit(NFS_INO_LAYOUTCOMMIT, &NFS_I(inode)->flags)) {
  1413. int status;
  1414. bool sync = true;
  1415. if (wbc->sync_mode == WB_SYNC_NONE)
  1416. sync = false;
  1417. status = pnfs_layoutcommit_inode(inode, sync);
  1418. if (status < 0)
  1419. return status;
  1420. }
  1421. return ret;
  1422. }
  1423. /*
  1424. * flush the inode to disk.
  1425. */
  1426. int nfs_wb_all(struct inode *inode)
  1427. {
  1428. struct writeback_control wbc = {
  1429. .sync_mode = WB_SYNC_ALL,
  1430. .nr_to_write = LONG_MAX,
  1431. .range_start = 0,
  1432. .range_end = LLONG_MAX,
  1433. };
  1434. return sync_inode(inode, &wbc);
  1435. }
  1436. int nfs_wb_page_cancel(struct inode *inode, struct page *page)
  1437. {
  1438. struct nfs_page *req;
  1439. int ret = 0;
  1440. BUG_ON(!PageLocked(page));
  1441. for (;;) {
  1442. wait_on_page_writeback(page);
  1443. req = nfs_page_find_request(page);
  1444. if (req == NULL)
  1445. break;
  1446. if (nfs_lock_request_dontget(req)) {
  1447. nfs_inode_remove_request(req);
  1448. /*
  1449. * In case nfs_inode_remove_request has marked the
  1450. * page as being dirty
  1451. */
  1452. cancel_dirty_page(page, PAGE_CACHE_SIZE);
  1453. nfs_unlock_request(req);
  1454. break;
  1455. }
  1456. ret = nfs_wait_on_request(req);
  1457. nfs_release_request(req);
  1458. if (ret < 0)
  1459. break;
  1460. }
  1461. return ret;
  1462. }
  1463. /*
  1464. * Write back all requests on one page - we do this before reading it.
  1465. */
  1466. int nfs_wb_page(struct inode *inode, struct page *page)
  1467. {
  1468. loff_t range_start = page_offset(page);
  1469. loff_t range_end = range_start + (loff_t)(PAGE_CACHE_SIZE - 1);
  1470. struct writeback_control wbc = {
  1471. .sync_mode = WB_SYNC_ALL,
  1472. .nr_to_write = 0,
  1473. .range_start = range_start,
  1474. .range_end = range_end,
  1475. };
  1476. int ret;
  1477. for (;;) {
  1478. wait_on_page_writeback(page);
  1479. if (clear_page_dirty_for_io(page)) {
  1480. ret = nfs_writepage_locked(page, &wbc);
  1481. if (ret < 0)
  1482. goto out_error;
  1483. continue;
  1484. }
  1485. if (!PagePrivate(page))
  1486. break;
  1487. ret = nfs_commit_inode(inode, FLUSH_SYNC);
  1488. if (ret < 0)
  1489. goto out_error;
  1490. }
  1491. return 0;
  1492. out_error:
  1493. return ret;
  1494. }
  1495. #ifdef CONFIG_MIGRATION
  1496. int nfs_migrate_page(struct address_space *mapping, struct page *newpage,
  1497. struct page *page)
  1498. {
  1499. /*
  1500. * If PagePrivate is set, then the page is currently associated with
  1501. * an in-progress read or write request. Don't try to migrate it.
  1502. *
  1503. * FIXME: we could do this in principle, but we'll need a way to ensure
  1504. * that we can safely release the inode reference while holding
  1505. * the page lock.
  1506. */
  1507. if (PagePrivate(page))
  1508. return -EBUSY;
  1509. nfs_fscache_release_page(page, GFP_KERNEL);
  1510. return migrate_page(mapping, newpage, page);
  1511. }
  1512. #endif
  1513. int __init nfs_init_writepagecache(void)
  1514. {
  1515. nfs_wdata_cachep = kmem_cache_create("nfs_write_data",
  1516. sizeof(struct nfs_write_data),
  1517. 0, SLAB_HWCACHE_ALIGN,
  1518. NULL);
  1519. if (nfs_wdata_cachep == NULL)
  1520. return -ENOMEM;
  1521. nfs_wdata_mempool = mempool_create_slab_pool(MIN_POOL_WRITE,
  1522. nfs_wdata_cachep);
  1523. if (nfs_wdata_mempool == NULL)
  1524. return -ENOMEM;
  1525. nfs_commit_mempool = mempool_create_slab_pool(MIN_POOL_COMMIT,
  1526. nfs_wdata_cachep);
  1527. if (nfs_commit_mempool == NULL)
  1528. return -ENOMEM;
  1529. /*
  1530. * NFS congestion size, scale with available memory.
  1531. *
  1532. * 64MB: 8192k
  1533. * 128MB: 11585k
  1534. * 256MB: 16384k
  1535. * 512MB: 23170k
  1536. * 1GB: 32768k
  1537. * 2GB: 46340k
  1538. * 4GB: 65536k
  1539. * 8GB: 92681k
  1540. * 16GB: 131072k
  1541. *
  1542. * This allows larger machines to have larger/more transfers.
  1543. * Limit the default to 256M
  1544. */
  1545. nfs_congestion_kb = (16*int_sqrt(totalram_pages)) << (PAGE_SHIFT-10);
  1546. if (nfs_congestion_kb > 256*1024)
  1547. nfs_congestion_kb = 256*1024;
  1548. return 0;
  1549. }
  1550. void nfs_destroy_writepagecache(void)
  1551. {
  1552. mempool_destroy(nfs_commit_mempool);
  1553. mempool_destroy(nfs_wdata_mempool);
  1554. kmem_cache_destroy(nfs_wdata_cachep);
  1555. }