direct.c 27 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013
  1. /*
  2. * linux/fs/nfs/direct.c
  3. *
  4. * Copyright (C) 2003 by Chuck Lever <cel@netapp.com>
  5. *
  6. * High-performance uncached I/O for the Linux NFS client
  7. *
  8. * There are important applications whose performance or correctness
  9. * depends on uncached access to file data. Database clusters
  10. * (multiple copies of the same instance running on separate hosts)
  11. * implement their own cache coherency protocol that subsumes file
  12. * system cache protocols. Applications that process datasets
  13. * considerably larger than the client's memory do not always benefit
  14. * from a local cache. A streaming video server, for instance, has no
  15. * need to cache the contents of a file.
  16. *
  17. * When an application requests uncached I/O, all read and write requests
  18. * are made directly to the server; data stored or fetched via these
  19. * requests is not cached in the Linux page cache. The client does not
  20. * correct unaligned requests from applications. All requested bytes are
  21. * held on permanent storage before a direct write system call returns to
  22. * an application.
  23. *
  24. * Solaris implements an uncached I/O facility called directio() that
  25. * is used for backups and sequential I/O to very large files. Solaris
  26. * also supports uncaching whole NFS partitions with "-o forcedirectio,"
  27. * an undocumented mount option.
  28. *
  29. * Designed by Jeff Kimmel, Chuck Lever, and Trond Myklebust, with
  30. * help from Andrew Morton.
  31. *
  32. * 18 Dec 2001 Initial implementation for 2.4 --cel
  33. * 08 Jul 2002 Version for 2.4.19, with bug fixes --trondmy
  34. * 08 Jun 2003 Port to 2.5 APIs --cel
  35. * 31 Mar 2004 Handle direct I/O without VFS support --cel
  36. * 15 Sep 2004 Parallel async reads --cel
  37. * 04 May 2005 support O_DIRECT with aio --cel
  38. *
  39. */
  40. #include <linux/errno.h>
  41. #include <linux/sched.h>
  42. #include <linux/kernel.h>
  43. #include <linux/file.h>
  44. #include <linux/pagemap.h>
  45. #include <linux/kref.h>
  46. #include <linux/nfs_fs.h>
  47. #include <linux/nfs_page.h>
  48. #include <linux/sunrpc/clnt.h>
  49. #include <asm/system.h>
  50. #include <asm/uaccess.h>
  51. #include <asm/atomic.h>
  52. #include "internal.h"
  53. #include "iostat.h"
  54. #define NFSDBG_FACILITY NFSDBG_VFS
  55. static struct kmem_cache *nfs_direct_cachep;
  56. /*
  57. * This represents a set of asynchronous requests that we're waiting on
  58. */
  59. struct nfs_direct_req {
  60. struct kref kref; /* release manager */
  61. /* I/O parameters */
  62. struct nfs_open_context *ctx; /* file open context info */
  63. struct kiocb * iocb; /* controlling i/o request */
  64. struct inode * inode; /* target file of i/o */
  65. /* completion state */
  66. atomic_t io_count; /* i/os we're waiting for */
  67. spinlock_t lock; /* protect completion state */
  68. ssize_t count, /* bytes actually processed */
  69. error; /* any reported error */
  70. struct completion completion; /* wait for i/o completion */
  71. /* commit state */
  72. struct list_head rewrite_list; /* saved nfs_write_data structs */
  73. struct nfs_write_data * commit_data; /* special write_data for commits */
  74. int flags;
  75. #define NFS_ODIRECT_DO_COMMIT (1) /* an unstable reply was received */
  76. #define NFS_ODIRECT_RESCHED_WRITES (2) /* write verification failed */
  77. struct nfs_writeverf verf; /* unstable write verifier */
  78. };
  79. static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode);
  80. static const struct rpc_call_ops nfs_write_direct_ops;
  81. static inline void get_dreq(struct nfs_direct_req *dreq)
  82. {
  83. atomic_inc(&dreq->io_count);
  84. }
  85. static inline int put_dreq(struct nfs_direct_req *dreq)
  86. {
  87. return atomic_dec_and_test(&dreq->io_count);
  88. }
  89. /**
  90. * nfs_direct_IO - NFS address space operation for direct I/O
  91. * @rw: direction (read or write)
  92. * @iocb: target I/O control block
  93. * @iov: array of vectors that define I/O buffer
  94. * @pos: offset in file to begin the operation
  95. * @nr_segs: size of iovec array
  96. *
  97. * The presence of this routine in the address space ops vector means
  98. * the NFS client supports direct I/O. However, we shunt off direct
  99. * read and write requests before the VFS gets them, so this method
  100. * should never be called.
  101. */
  102. ssize_t nfs_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, loff_t pos, unsigned long nr_segs)
  103. {
  104. dprintk("NFS: nfs_direct_IO (%s) off/no(%Ld/%lu) EINVAL\n",
  105. iocb->ki_filp->f_path.dentry->d_name.name,
  106. (long long) pos, nr_segs);
  107. return -EINVAL;
  108. }
  109. static void nfs_direct_dirty_pages(struct page **pages, unsigned int pgbase, size_t count)
  110. {
  111. unsigned int npages;
  112. unsigned int i;
  113. if (count == 0)
  114. return;
  115. pages += (pgbase >> PAGE_SHIFT);
  116. npages = (count + (pgbase & ~PAGE_MASK) + PAGE_SIZE - 1) >> PAGE_SHIFT;
  117. for (i = 0; i < npages; i++) {
  118. struct page *page = pages[i];
  119. if (!PageCompound(page))
  120. set_page_dirty(page);
  121. }
  122. }
  123. static void nfs_direct_release_pages(struct page **pages, unsigned int npages)
  124. {
  125. unsigned int i;
  126. for (i = 0; i < npages; i++)
  127. page_cache_release(pages[i]);
  128. }
  129. static inline struct nfs_direct_req *nfs_direct_req_alloc(void)
  130. {
  131. struct nfs_direct_req *dreq;
  132. dreq = kmem_cache_alloc(nfs_direct_cachep, GFP_KERNEL);
  133. if (!dreq)
  134. return NULL;
  135. kref_init(&dreq->kref);
  136. kref_get(&dreq->kref);
  137. init_completion(&dreq->completion);
  138. INIT_LIST_HEAD(&dreq->rewrite_list);
  139. dreq->iocb = NULL;
  140. dreq->ctx = NULL;
  141. spin_lock_init(&dreq->lock);
  142. atomic_set(&dreq->io_count, 0);
  143. dreq->count = 0;
  144. dreq->error = 0;
  145. dreq->flags = 0;
  146. return dreq;
  147. }
  148. static void nfs_direct_req_free(struct kref *kref)
  149. {
  150. struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref);
  151. if (dreq->ctx != NULL)
  152. put_nfs_open_context(dreq->ctx);
  153. kmem_cache_free(nfs_direct_cachep, dreq);
  154. }
  155. static void nfs_direct_req_release(struct nfs_direct_req *dreq)
  156. {
  157. kref_put(&dreq->kref, nfs_direct_req_free);
  158. }
  159. /*
  160. * Collects and returns the final error value/byte-count.
  161. */
  162. static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)
  163. {
  164. ssize_t result = -EIOCBQUEUED;
  165. /* Async requests don't wait here */
  166. if (dreq->iocb)
  167. goto out;
  168. result = wait_for_completion_killable(&dreq->completion);
  169. if (!result)
  170. result = dreq->error;
  171. if (!result)
  172. result = dreq->count;
  173. out:
  174. return (ssize_t) result;
  175. }
  176. /*
  177. * Synchronous I/O uses a stack-allocated iocb. Thus we can't trust
  178. * the iocb is still valid here if this is a synchronous request.
  179. */
  180. static void nfs_direct_complete(struct nfs_direct_req *dreq)
  181. {
  182. if (dreq->iocb) {
  183. long res = (long) dreq->error;
  184. if (!res)
  185. res = (long) dreq->count;
  186. aio_complete(dreq->iocb, res, 0);
  187. }
  188. complete_all(&dreq->completion);
  189. nfs_direct_req_release(dreq);
  190. }
  191. /*
  192. * We must hold a reference to all the pages in this direct read request
  193. * until the RPCs complete. This could be long *after* we are woken up in
  194. * nfs_direct_wait (for instance, if someone hits ^C on a slow server).
  195. */
  196. static void nfs_direct_read_result(struct rpc_task *task, void *calldata)
  197. {
  198. struct nfs_read_data *data = calldata;
  199. nfs_readpage_result(task, data);
  200. }
  201. static void nfs_direct_read_release(void *calldata)
  202. {
  203. struct nfs_read_data *data = calldata;
  204. struct nfs_direct_req *dreq = (struct nfs_direct_req *) data->req;
  205. int status = data->task.tk_status;
  206. spin_lock(&dreq->lock);
  207. if (unlikely(status < 0)) {
  208. dreq->error = status;
  209. spin_unlock(&dreq->lock);
  210. } else {
  211. dreq->count += data->res.count;
  212. spin_unlock(&dreq->lock);
  213. nfs_direct_dirty_pages(data->pagevec,
  214. data->args.pgbase,
  215. data->res.count);
  216. }
  217. nfs_direct_release_pages(data->pagevec, data->npages);
  218. if (put_dreq(dreq))
  219. nfs_direct_complete(dreq);
  220. nfs_readdata_free(data);
  221. }
  222. static const struct rpc_call_ops nfs_read_direct_ops = {
  223. #if defined(CONFIG_NFS_V4_1)
  224. .rpc_call_prepare = nfs_read_prepare,
  225. #endif /* CONFIG_NFS_V4_1 */
  226. .rpc_call_done = nfs_direct_read_result,
  227. .rpc_release = nfs_direct_read_release,
  228. };
  229. /*
  230. * For each rsize'd chunk of the user's buffer, dispatch an NFS READ
  231. * operation. If nfs_readdata_alloc() or get_user_pages() fails,
  232. * bail and stop sending more reads. Read length accounting is
  233. * handled automatically by nfs_direct_read_result(). Otherwise, if
  234. * no requests have been sent, just return an error.
  235. */
  236. static ssize_t nfs_direct_read_schedule_segment(struct nfs_direct_req *dreq,
  237. const struct iovec *iov,
  238. loff_t pos)
  239. {
  240. struct nfs_open_context *ctx = dreq->ctx;
  241. struct inode *inode = ctx->path.dentry->d_inode;
  242. unsigned long user_addr = (unsigned long)iov->iov_base;
  243. size_t count = iov->iov_len;
  244. size_t rsize = NFS_SERVER(inode)->rsize;
  245. struct rpc_task *task;
  246. struct rpc_message msg = {
  247. .rpc_cred = ctx->cred,
  248. };
  249. struct rpc_task_setup task_setup_data = {
  250. .rpc_client = NFS_CLIENT(inode),
  251. .rpc_message = &msg,
  252. .callback_ops = &nfs_read_direct_ops,
  253. .workqueue = nfsiod_workqueue,
  254. .flags = RPC_TASK_ASYNC,
  255. };
  256. unsigned int pgbase;
  257. int result;
  258. ssize_t started = 0;
  259. do {
  260. struct nfs_read_data *data;
  261. size_t bytes;
  262. pgbase = user_addr & ~PAGE_MASK;
  263. bytes = min(rsize,count);
  264. result = -ENOMEM;
  265. data = nfs_readdata_alloc(nfs_page_array_len(pgbase, bytes));
  266. if (unlikely(!data))
  267. break;
  268. down_read(&current->mm->mmap_sem);
  269. result = get_user_pages(current, current->mm, user_addr,
  270. data->npages, 1, 0, data->pagevec, NULL);
  271. up_read(&current->mm->mmap_sem);
  272. if (result < 0) {
  273. nfs_readdata_free(data);
  274. break;
  275. }
  276. if ((unsigned)result < data->npages) {
  277. bytes = result * PAGE_SIZE;
  278. if (bytes <= pgbase) {
  279. nfs_direct_release_pages(data->pagevec, result);
  280. nfs_readdata_free(data);
  281. break;
  282. }
  283. bytes -= pgbase;
  284. data->npages = result;
  285. }
  286. get_dreq(dreq);
  287. data->req = (struct nfs_page *) dreq;
  288. data->inode = inode;
  289. data->cred = msg.rpc_cred;
  290. data->args.fh = NFS_FH(inode);
  291. data->args.context = ctx;
  292. data->args.offset = pos;
  293. data->args.pgbase = pgbase;
  294. data->args.pages = data->pagevec;
  295. data->args.count = bytes;
  296. data->res.fattr = &data->fattr;
  297. data->res.eof = 0;
  298. data->res.count = bytes;
  299. nfs_fattr_init(&data->fattr);
  300. msg.rpc_argp = &data->args;
  301. msg.rpc_resp = &data->res;
  302. task_setup_data.task = &data->task;
  303. task_setup_data.callback_data = data;
  304. NFS_PROTO(inode)->read_setup(data, &msg);
  305. task = rpc_run_task(&task_setup_data);
  306. if (IS_ERR(task))
  307. break;
  308. rpc_put_task(task);
  309. dprintk("NFS: %5u initiated direct read call "
  310. "(req %s/%Ld, %zu bytes @ offset %Lu)\n",
  311. data->task.tk_pid,
  312. inode->i_sb->s_id,
  313. (long long)NFS_FILEID(inode),
  314. bytes,
  315. (unsigned long long)data->args.offset);
  316. started += bytes;
  317. user_addr += bytes;
  318. pos += bytes;
  319. /* FIXME: Remove this unnecessary math from final patch */
  320. pgbase += bytes;
  321. pgbase &= ~PAGE_MASK;
  322. BUG_ON(pgbase != (user_addr & ~PAGE_MASK));
  323. count -= bytes;
  324. } while (count != 0);
  325. if (started)
  326. return started;
  327. return result < 0 ? (ssize_t) result : -EFAULT;
  328. }
  329. static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq,
  330. const struct iovec *iov,
  331. unsigned long nr_segs,
  332. loff_t pos)
  333. {
  334. ssize_t result = -EINVAL;
  335. size_t requested_bytes = 0;
  336. unsigned long seg;
  337. get_dreq(dreq);
  338. for (seg = 0; seg < nr_segs; seg++) {
  339. const struct iovec *vec = &iov[seg];
  340. result = nfs_direct_read_schedule_segment(dreq, vec, pos);
  341. if (result < 0)
  342. break;
  343. requested_bytes += result;
  344. if ((size_t)result < vec->iov_len)
  345. break;
  346. pos += vec->iov_len;
  347. }
  348. if (put_dreq(dreq))
  349. nfs_direct_complete(dreq);
  350. if (requested_bytes != 0)
  351. return 0;
  352. if (result < 0)
  353. return result;
  354. return -EIO;
  355. }
  356. static ssize_t nfs_direct_read(struct kiocb *iocb, const struct iovec *iov,
  357. unsigned long nr_segs, loff_t pos)
  358. {
  359. ssize_t result = 0;
  360. struct inode *inode = iocb->ki_filp->f_mapping->host;
  361. struct nfs_direct_req *dreq;
  362. dreq = nfs_direct_req_alloc();
  363. if (!dreq)
  364. return -ENOMEM;
  365. dreq->inode = inode;
  366. dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
  367. if (!is_sync_kiocb(iocb))
  368. dreq->iocb = iocb;
  369. result = nfs_direct_read_schedule_iovec(dreq, iov, nr_segs, pos);
  370. if (!result)
  371. result = nfs_direct_wait(dreq);
  372. nfs_direct_req_release(dreq);
  373. return result;
  374. }
  375. static void nfs_direct_free_writedata(struct nfs_direct_req *dreq)
  376. {
  377. while (!list_empty(&dreq->rewrite_list)) {
  378. struct nfs_write_data *data = list_entry(dreq->rewrite_list.next, struct nfs_write_data, pages);
  379. list_del(&data->pages);
  380. nfs_direct_release_pages(data->pagevec, data->npages);
  381. nfs_writedata_free(data);
  382. }
  383. }
  384. #if defined(CONFIG_NFS_V3) || defined(CONFIG_NFS_V4)
  385. static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)
  386. {
  387. struct inode *inode = dreq->inode;
  388. struct list_head *p;
  389. struct nfs_write_data *data;
  390. struct rpc_task *task;
  391. struct rpc_message msg = {
  392. .rpc_cred = dreq->ctx->cred,
  393. };
  394. struct rpc_task_setup task_setup_data = {
  395. .rpc_client = NFS_CLIENT(inode),
  396. .rpc_message = &msg,
  397. .callback_ops = &nfs_write_direct_ops,
  398. .workqueue = nfsiod_workqueue,
  399. .flags = RPC_TASK_ASYNC,
  400. };
  401. dreq->count = 0;
  402. get_dreq(dreq);
  403. list_for_each(p, &dreq->rewrite_list) {
  404. data = list_entry(p, struct nfs_write_data, pages);
  405. get_dreq(dreq);
  406. /* Use stable writes */
  407. data->args.stable = NFS_FILE_SYNC;
  408. /*
  409. * Reset data->res.
  410. */
  411. nfs_fattr_init(&data->fattr);
  412. data->res.count = data->args.count;
  413. memset(&data->verf, 0, sizeof(data->verf));
  414. /*
  415. * Reuse data->task; data->args should not have changed
  416. * since the original request was sent.
  417. */
  418. task_setup_data.task = &data->task;
  419. task_setup_data.callback_data = data;
  420. msg.rpc_argp = &data->args;
  421. msg.rpc_resp = &data->res;
  422. NFS_PROTO(inode)->write_setup(data, &msg);
  423. /*
  424. * We're called via an RPC callback, so BKL is already held.
  425. */
  426. task = rpc_run_task(&task_setup_data);
  427. if (!IS_ERR(task))
  428. rpc_put_task(task);
  429. dprintk("NFS: %5u rescheduled direct write call (req %s/%Ld, %u bytes @ offset %Lu)\n",
  430. data->task.tk_pid,
  431. inode->i_sb->s_id,
  432. (long long)NFS_FILEID(inode),
  433. data->args.count,
  434. (unsigned long long)data->args.offset);
  435. }
  436. if (put_dreq(dreq))
  437. nfs_direct_write_complete(dreq, inode);
  438. }
  439. static void nfs_direct_commit_result(struct rpc_task *task, void *calldata)
  440. {
  441. struct nfs_write_data *data = calldata;
  442. /* Call the NFS version-specific code */
  443. NFS_PROTO(data->inode)->commit_done(task, data);
  444. }
  445. static void nfs_direct_commit_release(void *calldata)
  446. {
  447. struct nfs_write_data *data = calldata;
  448. struct nfs_direct_req *dreq = (struct nfs_direct_req *) data->req;
  449. int status = data->task.tk_status;
  450. if (status < 0) {
  451. dprintk("NFS: %5u commit failed with error %d.\n",
  452. data->task.tk_pid, status);
  453. dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
  454. } else if (memcmp(&dreq->verf, &data->verf, sizeof(data->verf))) {
  455. dprintk("NFS: %5u commit verify failed\n", data->task.tk_pid);
  456. dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
  457. }
  458. dprintk("NFS: %5u commit returned %d\n", data->task.tk_pid, status);
  459. nfs_direct_write_complete(dreq, data->inode);
  460. nfs_commit_free(data);
  461. }
  462. static const struct rpc_call_ops nfs_commit_direct_ops = {
  463. #if defined(CONFIG_NFS_V4_1)
  464. .rpc_call_prepare = nfs_write_prepare,
  465. #endif /* CONFIG_NFS_V4_1 */
  466. .rpc_call_done = nfs_direct_commit_result,
  467. .rpc_release = nfs_direct_commit_release,
  468. };
  469. static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)
  470. {
  471. struct nfs_write_data *data = dreq->commit_data;
  472. struct rpc_task *task;
  473. struct rpc_message msg = {
  474. .rpc_argp = &data->args,
  475. .rpc_resp = &data->res,
  476. .rpc_cred = dreq->ctx->cred,
  477. };
  478. struct rpc_task_setup task_setup_data = {
  479. .task = &data->task,
  480. .rpc_client = NFS_CLIENT(dreq->inode),
  481. .rpc_message = &msg,
  482. .callback_ops = &nfs_commit_direct_ops,
  483. .callback_data = data,
  484. .workqueue = nfsiod_workqueue,
  485. .flags = RPC_TASK_ASYNC,
  486. };
  487. data->inode = dreq->inode;
  488. data->cred = msg.rpc_cred;
  489. data->args.fh = NFS_FH(data->inode);
  490. data->args.offset = 0;
  491. data->args.count = 0;
  492. data->args.context = dreq->ctx;
  493. data->res.count = 0;
  494. data->res.fattr = &data->fattr;
  495. data->res.verf = &data->verf;
  496. nfs_fattr_init(&data->fattr);
  497. NFS_PROTO(data->inode)->commit_setup(data, &msg);
  498. /* Note: task.tk_ops->rpc_release will free dreq->commit_data */
  499. dreq->commit_data = NULL;
  500. dprintk("NFS: %5u initiated commit call\n", data->task.tk_pid);
  501. task = rpc_run_task(&task_setup_data);
  502. if (!IS_ERR(task))
  503. rpc_put_task(task);
  504. }
  505. static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode)
  506. {
  507. int flags = dreq->flags;
  508. dreq->flags = 0;
  509. switch (flags) {
  510. case NFS_ODIRECT_DO_COMMIT:
  511. nfs_direct_commit_schedule(dreq);
  512. break;
  513. case NFS_ODIRECT_RESCHED_WRITES:
  514. nfs_direct_write_reschedule(dreq);
  515. break;
  516. default:
  517. if (dreq->commit_data != NULL)
  518. nfs_commit_free(dreq->commit_data);
  519. nfs_direct_free_writedata(dreq);
  520. nfs_zap_mapping(inode, inode->i_mapping);
  521. nfs_direct_complete(dreq);
  522. }
  523. }
  524. static void nfs_alloc_commit_data(struct nfs_direct_req *dreq)
  525. {
  526. dreq->commit_data = nfs_commitdata_alloc();
  527. if (dreq->commit_data != NULL)
  528. dreq->commit_data->req = (struct nfs_page *) dreq;
  529. }
  530. #else
  531. static inline void nfs_alloc_commit_data(struct nfs_direct_req *dreq)
  532. {
  533. dreq->commit_data = NULL;
  534. }
  535. static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode)
  536. {
  537. nfs_direct_free_writedata(dreq);
  538. nfs_zap_mapping(inode, inode->i_mapping);
  539. nfs_direct_complete(dreq);
  540. }
  541. #endif
  542. static void nfs_direct_write_result(struct rpc_task *task, void *calldata)
  543. {
  544. struct nfs_write_data *data = calldata;
  545. if (nfs_writeback_done(task, data) != 0)
  546. return;
  547. }
  548. /*
  549. * NB: Return the value of the first error return code. Subsequent
  550. * errors after the first one are ignored.
  551. */
  552. static void nfs_direct_write_release(void *calldata)
  553. {
  554. struct nfs_write_data *data = calldata;
  555. struct nfs_direct_req *dreq = (struct nfs_direct_req *) data->req;
  556. int status = data->task.tk_status;
  557. spin_lock(&dreq->lock);
  558. if (unlikely(status < 0)) {
  559. /* An error has occurred, so we should not commit */
  560. dreq->flags = 0;
  561. dreq->error = status;
  562. }
  563. if (unlikely(dreq->error != 0))
  564. goto out_unlock;
  565. dreq->count += data->res.count;
  566. if (data->res.verf->committed != NFS_FILE_SYNC) {
  567. switch (dreq->flags) {
  568. case 0:
  569. memcpy(&dreq->verf, &data->verf, sizeof(dreq->verf));
  570. dreq->flags = NFS_ODIRECT_DO_COMMIT;
  571. break;
  572. case NFS_ODIRECT_DO_COMMIT:
  573. if (memcmp(&dreq->verf, &data->verf, sizeof(dreq->verf))) {
  574. dprintk("NFS: %5u write verify failed\n", data->task.tk_pid);
  575. dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
  576. }
  577. }
  578. }
  579. out_unlock:
  580. spin_unlock(&dreq->lock);
  581. if (put_dreq(dreq))
  582. nfs_direct_write_complete(dreq, data->inode);
  583. }
  584. static const struct rpc_call_ops nfs_write_direct_ops = {
  585. #if defined(CONFIG_NFS_V4_1)
  586. .rpc_call_prepare = nfs_write_prepare,
  587. #endif /* CONFIG_NFS_V4_1 */
  588. .rpc_call_done = nfs_direct_write_result,
  589. .rpc_release = nfs_direct_write_release,
  590. };
  591. /*
  592. * For each wsize'd chunk of the user's buffer, dispatch an NFS WRITE
  593. * operation. If nfs_writedata_alloc() or get_user_pages() fails,
  594. * bail and stop sending more writes. Write length accounting is
  595. * handled automatically by nfs_direct_write_result(). Otherwise, if
  596. * no requests have been sent, just return an error.
  597. */
  598. static ssize_t nfs_direct_write_schedule_segment(struct nfs_direct_req *dreq,
  599. const struct iovec *iov,
  600. loff_t pos, int sync)
  601. {
  602. struct nfs_open_context *ctx = dreq->ctx;
  603. struct inode *inode = ctx->path.dentry->d_inode;
  604. unsigned long user_addr = (unsigned long)iov->iov_base;
  605. size_t count = iov->iov_len;
  606. struct rpc_task *task;
  607. struct rpc_message msg = {
  608. .rpc_cred = ctx->cred,
  609. };
  610. struct rpc_task_setup task_setup_data = {
  611. .rpc_client = NFS_CLIENT(inode),
  612. .rpc_message = &msg,
  613. .callback_ops = &nfs_write_direct_ops,
  614. .workqueue = nfsiod_workqueue,
  615. .flags = RPC_TASK_ASYNC,
  616. };
  617. size_t wsize = NFS_SERVER(inode)->wsize;
  618. unsigned int pgbase;
  619. int result;
  620. ssize_t started = 0;
  621. do {
  622. struct nfs_write_data *data;
  623. size_t bytes;
  624. pgbase = user_addr & ~PAGE_MASK;
  625. bytes = min(wsize,count);
  626. result = -ENOMEM;
  627. data = nfs_writedata_alloc(nfs_page_array_len(pgbase, bytes));
  628. if (unlikely(!data))
  629. break;
  630. down_read(&current->mm->mmap_sem);
  631. result = get_user_pages(current, current->mm, user_addr,
  632. data->npages, 0, 0, data->pagevec, NULL);
  633. up_read(&current->mm->mmap_sem);
  634. if (result < 0) {
  635. nfs_writedata_free(data);
  636. break;
  637. }
  638. if ((unsigned)result < data->npages) {
  639. bytes = result * PAGE_SIZE;
  640. if (bytes <= pgbase) {
  641. nfs_direct_release_pages(data->pagevec, result);
  642. nfs_writedata_free(data);
  643. break;
  644. }
  645. bytes -= pgbase;
  646. data->npages = result;
  647. }
  648. get_dreq(dreq);
  649. list_move_tail(&data->pages, &dreq->rewrite_list);
  650. data->req = (struct nfs_page *) dreq;
  651. data->inode = inode;
  652. data->cred = msg.rpc_cred;
  653. data->args.fh = NFS_FH(inode);
  654. data->args.context = ctx;
  655. data->args.offset = pos;
  656. data->args.pgbase = pgbase;
  657. data->args.pages = data->pagevec;
  658. data->args.count = bytes;
  659. data->args.stable = sync;
  660. data->res.fattr = &data->fattr;
  661. data->res.count = bytes;
  662. data->res.verf = &data->verf;
  663. nfs_fattr_init(&data->fattr);
  664. task_setup_data.task = &data->task;
  665. task_setup_data.callback_data = data;
  666. msg.rpc_argp = &data->args;
  667. msg.rpc_resp = &data->res;
  668. NFS_PROTO(inode)->write_setup(data, &msg);
  669. task = rpc_run_task(&task_setup_data);
  670. if (IS_ERR(task))
  671. break;
  672. rpc_put_task(task);
  673. dprintk("NFS: %5u initiated direct write call "
  674. "(req %s/%Ld, %zu bytes @ offset %Lu)\n",
  675. data->task.tk_pid,
  676. inode->i_sb->s_id,
  677. (long long)NFS_FILEID(inode),
  678. bytes,
  679. (unsigned long long)data->args.offset);
  680. started += bytes;
  681. user_addr += bytes;
  682. pos += bytes;
  683. /* FIXME: Remove this useless math from the final patch */
  684. pgbase += bytes;
  685. pgbase &= ~PAGE_MASK;
  686. BUG_ON(pgbase != (user_addr & ~PAGE_MASK));
  687. count -= bytes;
  688. } while (count != 0);
  689. if (started)
  690. return started;
  691. return result < 0 ? (ssize_t) result : -EFAULT;
  692. }
  693. static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
  694. const struct iovec *iov,
  695. unsigned long nr_segs,
  696. loff_t pos, int sync)
  697. {
  698. ssize_t result = 0;
  699. size_t requested_bytes = 0;
  700. unsigned long seg;
  701. get_dreq(dreq);
  702. for (seg = 0; seg < nr_segs; seg++) {
  703. const struct iovec *vec = &iov[seg];
  704. result = nfs_direct_write_schedule_segment(dreq, vec,
  705. pos, sync);
  706. if (result < 0)
  707. break;
  708. requested_bytes += result;
  709. if ((size_t)result < vec->iov_len)
  710. break;
  711. pos += vec->iov_len;
  712. }
  713. if (put_dreq(dreq))
  714. nfs_direct_write_complete(dreq, dreq->inode);
  715. if (requested_bytes != 0)
  716. return 0;
  717. if (result < 0)
  718. return result;
  719. return -EIO;
  720. }
  721. static ssize_t nfs_direct_write(struct kiocb *iocb, const struct iovec *iov,
  722. unsigned long nr_segs, loff_t pos,
  723. size_t count)
  724. {
  725. ssize_t result = 0;
  726. struct inode *inode = iocb->ki_filp->f_mapping->host;
  727. struct nfs_direct_req *dreq;
  728. size_t wsize = NFS_SERVER(inode)->wsize;
  729. int sync = NFS_UNSTABLE;
  730. dreq = nfs_direct_req_alloc();
  731. if (!dreq)
  732. return -ENOMEM;
  733. nfs_alloc_commit_data(dreq);
  734. if (dreq->commit_data == NULL || count < wsize)
  735. sync = NFS_FILE_SYNC;
  736. dreq->inode = inode;
  737. dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
  738. if (!is_sync_kiocb(iocb))
  739. dreq->iocb = iocb;
  740. result = nfs_direct_write_schedule_iovec(dreq, iov, nr_segs, pos, sync);
  741. if (!result)
  742. result = nfs_direct_wait(dreq);
  743. nfs_direct_req_release(dreq);
  744. return result;
  745. }
  746. /**
  747. * nfs_file_direct_read - file direct read operation for NFS files
  748. * @iocb: target I/O control block
  749. * @iov: vector of user buffers into which to read data
  750. * @nr_segs: size of iov vector
  751. * @pos: byte offset in file where reading starts
  752. *
  753. * We use this function for direct reads instead of calling
  754. * generic_file_aio_read() in order to avoid gfar's check to see if
  755. * the request starts before the end of the file. For that check
  756. * to work, we must generate a GETATTR before each direct read, and
  757. * even then there is a window between the GETATTR and the subsequent
  758. * READ where the file size could change. Our preference is simply
  759. * to do all reads the application wants, and the server will take
  760. * care of managing the end of file boundary.
  761. *
  762. * This function also eliminates unnecessarily updating the file's
  763. * atime locally, as the NFS server sets the file's atime, and this
  764. * client must read the updated atime from the server back into its
  765. * cache.
  766. */
  767. ssize_t nfs_file_direct_read(struct kiocb *iocb, const struct iovec *iov,
  768. unsigned long nr_segs, loff_t pos)
  769. {
  770. ssize_t retval = -EINVAL;
  771. struct file *file = iocb->ki_filp;
  772. struct address_space *mapping = file->f_mapping;
  773. size_t count;
  774. count = iov_length(iov, nr_segs);
  775. nfs_add_stats(mapping->host, NFSIOS_DIRECTREADBYTES, count);
  776. dfprintk(FILE, "NFS: direct read(%s/%s, %zd@%Ld)\n",
  777. file->f_path.dentry->d_parent->d_name.name,
  778. file->f_path.dentry->d_name.name,
  779. count, (long long) pos);
  780. retval = 0;
  781. if (!count)
  782. goto out;
  783. retval = nfs_sync_mapping(mapping);
  784. if (retval)
  785. goto out;
  786. retval = nfs_direct_read(iocb, iov, nr_segs, pos);
  787. if (retval > 0)
  788. iocb->ki_pos = pos + retval;
  789. out:
  790. return retval;
  791. }
  792. /**
  793. * nfs_file_direct_write - file direct write operation for NFS files
  794. * @iocb: target I/O control block
  795. * @iov: vector of user buffers from which to write data
  796. * @nr_segs: size of iov vector
  797. * @pos: byte offset in file where writing starts
  798. *
  799. * We use this function for direct writes instead of calling
  800. * generic_file_aio_write() in order to avoid taking the inode
  801. * semaphore and updating the i_size. The NFS server will set
  802. * the new i_size and this client must read the updated size
  803. * back into its cache. We let the server do generic write
  804. * parameter checking and report problems.
  805. *
  806. * We eliminate local atime updates, see direct read above.
  807. *
  808. * We avoid unnecessary page cache invalidations for normal cached
  809. * readers of this file.
  810. *
  811. * Note that O_APPEND is not supported for NFS direct writes, as there
  812. * is no atomic O_APPEND write facility in the NFS protocol.
  813. */
  814. ssize_t nfs_file_direct_write(struct kiocb *iocb, const struct iovec *iov,
  815. unsigned long nr_segs, loff_t pos)
  816. {
  817. ssize_t retval = -EINVAL;
  818. struct file *file = iocb->ki_filp;
  819. struct address_space *mapping = file->f_mapping;
  820. size_t count;
  821. count = iov_length(iov, nr_segs);
  822. nfs_add_stats(mapping->host, NFSIOS_DIRECTWRITTENBYTES, count);
  823. dfprintk(FILE, "NFS: direct write(%s/%s, %zd@%Ld)\n",
  824. file->f_path.dentry->d_parent->d_name.name,
  825. file->f_path.dentry->d_name.name,
  826. count, (long long) pos);
  827. retval = generic_write_checks(file, &pos, &count, 0);
  828. if (retval)
  829. goto out;
  830. retval = -EINVAL;
  831. if ((ssize_t) count < 0)
  832. goto out;
  833. retval = 0;
  834. if (!count)
  835. goto out;
  836. retval = nfs_sync_mapping(mapping);
  837. if (retval)
  838. goto out;
  839. retval = nfs_direct_write(iocb, iov, nr_segs, pos, count);
  840. if (retval > 0)
  841. iocb->ki_pos = pos + retval;
  842. out:
  843. return retval;
  844. }
  845. /**
  846. * nfs_init_directcache - create a slab cache for nfs_direct_req structures
  847. *
  848. */
  849. int __init nfs_init_directcache(void)
  850. {
  851. nfs_direct_cachep = kmem_cache_create("nfs_direct_cache",
  852. sizeof(struct nfs_direct_req),
  853. 0, (SLAB_RECLAIM_ACCOUNT|
  854. SLAB_MEM_SPREAD),
  855. NULL);
  856. if (nfs_direct_cachep == NULL)
  857. return -ENOMEM;
  858. return 0;
  859. }
  860. /**
  861. * nfs_destroy_directcache - destroy the slab cache for nfs_direct_req structures
  862. *
  863. */
  864. void nfs_destroy_directcache(void)
  865. {
  866. kmem_cache_destroy(nfs_direct_cachep);
  867. }