read_write.c 27 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258
  1. /*
  2. * linux/fs/read_write.c
  3. *
  4. * Copyright (C) 1991, 1992 Linus Torvalds
  5. */
  6. #include <linux/slab.h>
  7. #include <linux/stat.h>
  8. #include <linux/fcntl.h>
  9. #include <linux/file.h>
  10. #include <linux/uio.h>
  11. #include <linux/aio.h>
  12. #include <linux/fsnotify.h>
  13. #include <linux/security.h>
  14. #include <linux/export.h>
  15. #include <linux/syscalls.h>
  16. #include <linux/pagemap.h>
  17. #include <linux/splice.h>
  18. #include <linux/compat.h>
  19. #include "internal.h"
  20. #include <asm/uaccess.h>
  21. #include <asm/unistd.h>
  22. typedef ssize_t (*io_fn_t)(struct file *, char __user *, size_t, loff_t *);
  23. typedef ssize_t (*iov_fn_t)(struct kiocb *, const struct iovec *,
  24. unsigned long, loff_t);
  25. const struct file_operations generic_ro_fops = {
  26. .llseek = generic_file_llseek,
  27. .read = do_sync_read,
  28. .aio_read = generic_file_aio_read,
  29. .mmap = generic_file_readonly_mmap,
  30. .splice_read = generic_file_splice_read,
  31. };
  32. EXPORT_SYMBOL(generic_ro_fops);
  33. static inline int unsigned_offsets(struct file *file)
  34. {
  35. return file->f_mode & FMODE_UNSIGNED_OFFSET;
  36. }
  37. /**
  38. * vfs_setpos - update the file offset for lseek
  39. * @file: file structure in question
  40. * @offset: file offset to seek to
  41. * @maxsize: maximum file size
  42. *
  43. * This is a low-level filesystem helper for updating the file offset to
  44. * the value specified by @offset if the given offset is valid and it is
  45. * not equal to the current file offset.
  46. *
  47. * Return the specified offset on success and -EINVAL on invalid offset.
  48. */
  49. loff_t vfs_setpos(struct file *file, loff_t offset, loff_t maxsize)
  50. {
  51. if (offset < 0 && !unsigned_offsets(file))
  52. return -EINVAL;
  53. if (offset > maxsize)
  54. return -EINVAL;
  55. if (offset != file->f_pos) {
  56. file->f_pos = offset;
  57. file->f_version = 0;
  58. }
  59. return offset;
  60. }
  61. EXPORT_SYMBOL(vfs_setpos);
  62. /**
  63. * generic_file_llseek_size - generic llseek implementation for regular files
  64. * @file: file structure to seek on
  65. * @offset: file offset to seek to
  66. * @whence: type of seek
  67. * @size: max size of this file in file system
  68. * @eof: offset used for SEEK_END position
  69. *
  70. * This is a variant of generic_file_llseek that allows passing in a custom
  71. * maximum file size and a custom EOF position, for e.g. hashed directories
  72. *
  73. * Synchronization:
  74. * SEEK_SET and SEEK_END are unsynchronized (but atomic on 64bit platforms)
  75. * SEEK_CUR is synchronized against other SEEK_CURs, but not read/writes.
  76. * read/writes behave like SEEK_SET against seeks.
  77. */
  78. loff_t
  79. generic_file_llseek_size(struct file *file, loff_t offset, int whence,
  80. loff_t maxsize, loff_t eof)
  81. {
  82. switch (whence) {
  83. case SEEK_END:
  84. offset += eof;
  85. break;
  86. case SEEK_CUR:
  87. /*
  88. * Here we special-case the lseek(fd, 0, SEEK_CUR)
  89. * position-querying operation. Avoid rewriting the "same"
  90. * f_pos value back to the file because a concurrent read(),
  91. * write() or lseek() might have altered it
  92. */
  93. if (offset == 0)
  94. return file->f_pos;
  95. /*
  96. * f_lock protects against read/modify/write race with other
  97. * SEEK_CURs. Note that parallel writes and reads behave
  98. * like SEEK_SET.
  99. */
  100. spin_lock(&file->f_lock);
  101. offset = vfs_setpos(file, file->f_pos + offset, maxsize);
  102. spin_unlock(&file->f_lock);
  103. return offset;
  104. case SEEK_DATA:
  105. /*
  106. * In the generic case the entire file is data, so as long as
  107. * offset isn't at the end of the file then the offset is data.
  108. */
  109. if (offset >= eof)
  110. return -ENXIO;
  111. break;
  112. case SEEK_HOLE:
  113. /*
  114. * There is a virtual hole at the end of the file, so as long as
  115. * offset isn't i_size or larger, return i_size.
  116. */
  117. if (offset >= eof)
  118. return -ENXIO;
  119. offset = eof;
  120. break;
  121. }
  122. return vfs_setpos(file, offset, maxsize);
  123. }
  124. EXPORT_SYMBOL(generic_file_llseek_size);
  125. /**
  126. * generic_file_llseek - generic llseek implementation for regular files
  127. * @file: file structure to seek on
  128. * @offset: file offset to seek to
  129. * @whence: type of seek
  130. *
  131. * This is a generic implemenation of ->llseek useable for all normal local
  132. * filesystems. It just updates the file offset to the value specified by
  133. * @offset and @whence.
  134. */
  135. loff_t generic_file_llseek(struct file *file, loff_t offset, int whence)
  136. {
  137. struct inode *inode = file->f_mapping->host;
  138. return generic_file_llseek_size(file, offset, whence,
  139. inode->i_sb->s_maxbytes,
  140. i_size_read(inode));
  141. }
  142. EXPORT_SYMBOL(generic_file_llseek);
  143. /**
  144. * fixed_size_llseek - llseek implementation for fixed-sized devices
  145. * @file: file structure to seek on
  146. * @offset: file offset to seek to
  147. * @whence: type of seek
  148. * @size: size of the file
  149. *
  150. */
  151. loff_t fixed_size_llseek(struct file *file, loff_t offset, int whence, loff_t size)
  152. {
  153. switch (whence) {
  154. case SEEK_SET: case SEEK_CUR: case SEEK_END:
  155. return generic_file_llseek_size(file, offset, whence,
  156. size, size);
  157. default:
  158. return -EINVAL;
  159. }
  160. }
  161. EXPORT_SYMBOL(fixed_size_llseek);
  162. /**
  163. * noop_llseek - No Operation Performed llseek implementation
  164. * @file: file structure to seek on
  165. * @offset: file offset to seek to
  166. * @whence: type of seek
  167. *
  168. * This is an implementation of ->llseek useable for the rare special case when
  169. * userspace expects the seek to succeed but the (device) file is actually not
  170. * able to perform the seek. In this case you use noop_llseek() instead of
  171. * falling back to the default implementation of ->llseek.
  172. */
  173. loff_t noop_llseek(struct file *file, loff_t offset, int whence)
  174. {
  175. return file->f_pos;
  176. }
  177. EXPORT_SYMBOL(noop_llseek);
  178. loff_t no_llseek(struct file *file, loff_t offset, int whence)
  179. {
  180. return -ESPIPE;
  181. }
  182. EXPORT_SYMBOL(no_llseek);
  183. loff_t default_llseek(struct file *file, loff_t offset, int whence)
  184. {
  185. struct inode *inode = file_inode(file);
  186. loff_t retval;
  187. mutex_lock(&inode->i_mutex);
  188. switch (whence) {
  189. case SEEK_END:
  190. offset += i_size_read(inode);
  191. break;
  192. case SEEK_CUR:
  193. if (offset == 0) {
  194. retval = file->f_pos;
  195. goto out;
  196. }
  197. offset += file->f_pos;
  198. break;
  199. case SEEK_DATA:
  200. /*
  201. * In the generic case the entire file is data, so as
  202. * long as offset isn't at the end of the file then the
  203. * offset is data.
  204. */
  205. if (offset >= inode->i_size) {
  206. retval = -ENXIO;
  207. goto out;
  208. }
  209. break;
  210. case SEEK_HOLE:
  211. /*
  212. * There is a virtual hole at the end of the file, so
  213. * as long as offset isn't i_size or larger, return
  214. * i_size.
  215. */
  216. if (offset >= inode->i_size) {
  217. retval = -ENXIO;
  218. goto out;
  219. }
  220. offset = inode->i_size;
  221. break;
  222. }
  223. retval = -EINVAL;
  224. if (offset >= 0 || unsigned_offsets(file)) {
  225. if (offset != file->f_pos) {
  226. file->f_pos = offset;
  227. file->f_version = 0;
  228. }
  229. retval = offset;
  230. }
  231. out:
  232. mutex_unlock(&inode->i_mutex);
  233. return retval;
  234. }
  235. EXPORT_SYMBOL(default_llseek);
  236. loff_t vfs_llseek(struct file *file, loff_t offset, int whence)
  237. {
  238. loff_t (*fn)(struct file *, loff_t, int);
  239. fn = no_llseek;
  240. if (file->f_mode & FMODE_LSEEK) {
  241. if (file->f_op->llseek)
  242. fn = file->f_op->llseek;
  243. }
  244. return fn(file, offset, whence);
  245. }
  246. EXPORT_SYMBOL(vfs_llseek);
  247. SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, whence)
  248. {
  249. off_t retval;
  250. struct fd f = fdget(fd);
  251. if (!f.file)
  252. return -EBADF;
  253. retval = -EINVAL;
  254. if (whence <= SEEK_MAX) {
  255. loff_t res = vfs_llseek(f.file, offset, whence);
  256. retval = res;
  257. if (res != (loff_t)retval)
  258. retval = -EOVERFLOW; /* LFS: should only happen on 32 bit platforms */
  259. }
  260. fdput(f);
  261. return retval;
  262. }
  263. #ifdef CONFIG_COMPAT
  264. COMPAT_SYSCALL_DEFINE3(lseek, unsigned int, fd, compat_off_t, offset, unsigned int, whence)
  265. {
  266. return sys_lseek(fd, offset, whence);
  267. }
  268. #endif
  269. #ifdef __ARCH_WANT_SYS_LLSEEK
  270. SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high,
  271. unsigned long, offset_low, loff_t __user *, result,
  272. unsigned int, whence)
  273. {
  274. int retval;
  275. struct fd f = fdget(fd);
  276. loff_t offset;
  277. if (!f.file)
  278. return -EBADF;
  279. retval = -EINVAL;
  280. if (whence > SEEK_MAX)
  281. goto out_putf;
  282. offset = vfs_llseek(f.file, ((loff_t) offset_high << 32) | offset_low,
  283. whence);
  284. retval = (int)offset;
  285. if (offset >= 0) {
  286. retval = -EFAULT;
  287. if (!copy_to_user(result, &offset, sizeof(offset)))
  288. retval = 0;
  289. }
  290. out_putf:
  291. fdput(f);
  292. return retval;
  293. }
  294. #endif
  295. /*
  296. * rw_verify_area doesn't like huge counts. We limit
  297. * them to something that fits in "int" so that others
  298. * won't have to do range checks all the time.
  299. */
  300. int rw_verify_area(int read_write, struct file *file, const loff_t *ppos, size_t count)
  301. {
  302. struct inode *inode;
  303. loff_t pos;
  304. int retval = -EINVAL;
  305. inode = file_inode(file);
  306. if (unlikely((ssize_t) count < 0))
  307. return retval;
  308. pos = *ppos;
  309. if (unlikely(pos < 0)) {
  310. if (!unsigned_offsets(file))
  311. return retval;
  312. if (count >= -pos) /* both values are in 0..LLONG_MAX */
  313. return -EOVERFLOW;
  314. } else if (unlikely((loff_t) (pos + count) < 0)) {
  315. if (!unsigned_offsets(file))
  316. return retval;
  317. }
  318. if (unlikely(inode->i_flock && mandatory_lock(inode))) {
  319. retval = locks_mandatory_area(
  320. read_write == READ ? FLOCK_VERIFY_READ : FLOCK_VERIFY_WRITE,
  321. inode, file, pos, count);
  322. if (retval < 0)
  323. return retval;
  324. }
  325. retval = security_file_permission(file,
  326. read_write == READ ? MAY_READ : MAY_WRITE);
  327. if (retval)
  328. return retval;
  329. return count > MAX_RW_COUNT ? MAX_RW_COUNT : count;
  330. }
  331. ssize_t do_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
  332. {
  333. struct iovec iov = { .iov_base = buf, .iov_len = len };
  334. struct kiocb kiocb;
  335. ssize_t ret;
  336. init_sync_kiocb(&kiocb, filp);
  337. kiocb.ki_pos = *ppos;
  338. kiocb.ki_nbytes = len;
  339. ret = filp->f_op->aio_read(&kiocb, &iov, 1, kiocb.ki_pos);
  340. if (-EIOCBQUEUED == ret)
  341. ret = wait_on_sync_kiocb(&kiocb);
  342. *ppos = kiocb.ki_pos;
  343. return ret;
  344. }
  345. EXPORT_SYMBOL(do_sync_read);
  346. ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
  347. {
  348. ssize_t ret;
  349. if (!(file->f_mode & FMODE_READ))
  350. return -EBADF;
  351. if (!file->f_op->read && !file->f_op->aio_read)
  352. return -EINVAL;
  353. if (unlikely(!access_ok(VERIFY_WRITE, buf, count)))
  354. return -EFAULT;
  355. ret = rw_verify_area(READ, file, pos, count);
  356. if (ret >= 0) {
  357. count = ret;
  358. if (file->f_op->read)
  359. ret = file->f_op->read(file, buf, count, pos);
  360. else
  361. ret = do_sync_read(file, buf, count, pos);
  362. if (ret > 0) {
  363. fsnotify_access(file);
  364. add_rchar(current, ret);
  365. }
  366. inc_syscr(current);
  367. }
  368. return ret;
  369. }
  370. EXPORT_SYMBOL(vfs_read);
  371. ssize_t do_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
  372. {
  373. struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
  374. struct kiocb kiocb;
  375. ssize_t ret;
  376. init_sync_kiocb(&kiocb, filp);
  377. kiocb.ki_pos = *ppos;
  378. kiocb.ki_nbytes = len;
  379. ret = filp->f_op->aio_write(&kiocb, &iov, 1, kiocb.ki_pos);
  380. if (-EIOCBQUEUED == ret)
  381. ret = wait_on_sync_kiocb(&kiocb);
  382. *ppos = kiocb.ki_pos;
  383. return ret;
  384. }
  385. EXPORT_SYMBOL(do_sync_write);
  386. ssize_t __kernel_write(struct file *file, const char *buf, size_t count, loff_t *pos)
  387. {
  388. mm_segment_t old_fs;
  389. const char __user *p;
  390. ssize_t ret;
  391. if (!file->f_op->write && !file->f_op->aio_write)
  392. return -EINVAL;
  393. old_fs = get_fs();
  394. set_fs(get_ds());
  395. p = (__force const char __user *)buf;
  396. if (count > MAX_RW_COUNT)
  397. count = MAX_RW_COUNT;
  398. if (file->f_op->write)
  399. ret = file->f_op->write(file, p, count, pos);
  400. else
  401. ret = do_sync_write(file, p, count, pos);
  402. set_fs(old_fs);
  403. if (ret > 0) {
  404. fsnotify_modify(file);
  405. add_wchar(current, ret);
  406. }
  407. inc_syscw(current);
  408. return ret;
  409. }
  410. ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
  411. {
  412. ssize_t ret;
  413. if (!(file->f_mode & FMODE_WRITE))
  414. return -EBADF;
  415. if (!file->f_op->write && !file->f_op->aio_write)
  416. return -EINVAL;
  417. if (unlikely(!access_ok(VERIFY_READ, buf, count)))
  418. return -EFAULT;
  419. ret = rw_verify_area(WRITE, file, pos, count);
  420. if (ret >= 0) {
  421. count = ret;
  422. file_start_write(file);
  423. if (file->f_op->write)
  424. ret = file->f_op->write(file, buf, count, pos);
  425. else
  426. ret = do_sync_write(file, buf, count, pos);
  427. if (ret > 0) {
  428. fsnotify_modify(file);
  429. add_wchar(current, ret);
  430. }
  431. inc_syscw(current);
  432. file_end_write(file);
  433. }
  434. return ret;
  435. }
  436. EXPORT_SYMBOL(vfs_write);
  437. static inline loff_t file_pos_read(struct file *file)
  438. {
  439. return file->f_pos;
  440. }
  441. static inline void file_pos_write(struct file *file, loff_t pos)
  442. {
  443. file->f_pos = pos;
  444. }
  445. SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count)
  446. {
  447. struct fd f = fdget(fd);
  448. ssize_t ret = -EBADF;
  449. if (f.file) {
  450. loff_t pos = file_pos_read(f.file);
  451. ret = vfs_read(f.file, buf, count, &pos);
  452. if (ret >= 0)
  453. file_pos_write(f.file, pos);
  454. fdput(f);
  455. }
  456. return ret;
  457. }
  458. SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf,
  459. size_t, count)
  460. {
  461. struct fd f = fdget(fd);
  462. ssize_t ret = -EBADF;
  463. if (f.file) {
  464. loff_t pos = file_pos_read(f.file);
  465. ret = vfs_write(f.file, buf, count, &pos);
  466. if (ret >= 0)
  467. file_pos_write(f.file, pos);
  468. fdput(f);
  469. }
  470. return ret;
  471. }
  472. SYSCALL_DEFINE4(pread64, unsigned int, fd, char __user *, buf,
  473. size_t, count, loff_t, pos)
  474. {
  475. struct fd f;
  476. ssize_t ret = -EBADF;
  477. if (pos < 0)
  478. return -EINVAL;
  479. f = fdget(fd);
  480. if (f.file) {
  481. ret = -ESPIPE;
  482. if (f.file->f_mode & FMODE_PREAD)
  483. ret = vfs_read(f.file, buf, count, &pos);
  484. fdput(f);
  485. }
  486. return ret;
  487. }
  488. SYSCALL_DEFINE4(pwrite64, unsigned int, fd, const char __user *, buf,
  489. size_t, count, loff_t, pos)
  490. {
  491. struct fd f;
  492. ssize_t ret = -EBADF;
  493. if (pos < 0)
  494. return -EINVAL;
  495. f = fdget(fd);
  496. if (f.file) {
  497. ret = -ESPIPE;
  498. if (f.file->f_mode & FMODE_PWRITE)
  499. ret = vfs_write(f.file, buf, count, &pos);
  500. fdput(f);
  501. }
  502. return ret;
  503. }
  504. /*
  505. * Reduce an iovec's length in-place. Return the resulting number of segments
  506. */
  507. unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to)
  508. {
  509. unsigned long seg = 0;
  510. size_t len = 0;
  511. while (seg < nr_segs) {
  512. seg++;
  513. if (len + iov->iov_len >= to) {
  514. iov->iov_len = to - len;
  515. break;
  516. }
  517. len += iov->iov_len;
  518. iov++;
  519. }
  520. return seg;
  521. }
  522. EXPORT_SYMBOL(iov_shorten);
  523. static ssize_t do_sync_readv_writev(struct file *filp, const struct iovec *iov,
  524. unsigned long nr_segs, size_t len, loff_t *ppos, iov_fn_t fn)
  525. {
  526. struct kiocb kiocb;
  527. ssize_t ret;
  528. init_sync_kiocb(&kiocb, filp);
  529. kiocb.ki_pos = *ppos;
  530. kiocb.ki_nbytes = len;
  531. ret = fn(&kiocb, iov, nr_segs, kiocb.ki_pos);
  532. if (ret == -EIOCBQUEUED)
  533. ret = wait_on_sync_kiocb(&kiocb);
  534. *ppos = kiocb.ki_pos;
  535. return ret;
  536. }
  537. /* Do it by hand, with file-ops */
  538. static ssize_t do_loop_readv_writev(struct file *filp, struct iovec *iov,
  539. unsigned long nr_segs, loff_t *ppos, io_fn_t fn)
  540. {
  541. struct iovec *vector = iov;
  542. ssize_t ret = 0;
  543. while (nr_segs > 0) {
  544. void __user *base;
  545. size_t len;
  546. ssize_t nr;
  547. base = vector->iov_base;
  548. len = vector->iov_len;
  549. vector++;
  550. nr_segs--;
  551. nr = fn(filp, base, len, ppos);
  552. if (nr < 0) {
  553. if (!ret)
  554. ret = nr;
  555. break;
  556. }
  557. ret += nr;
  558. if (nr != len)
  559. break;
  560. }
  561. return ret;
  562. }
  563. /* A write operation does a read from user space and vice versa */
  564. #define vrfy_dir(type) ((type) == READ ? VERIFY_WRITE : VERIFY_READ)
  565. ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector,
  566. unsigned long nr_segs, unsigned long fast_segs,
  567. struct iovec *fast_pointer,
  568. struct iovec **ret_pointer)
  569. {
  570. unsigned long seg;
  571. ssize_t ret;
  572. struct iovec *iov = fast_pointer;
  573. /*
  574. * SuS says "The readv() function *may* fail if the iovcnt argument
  575. * was less than or equal to 0, or greater than {IOV_MAX}. Linux has
  576. * traditionally returned zero for zero segments, so...
  577. */
  578. if (nr_segs == 0) {
  579. ret = 0;
  580. goto out;
  581. }
  582. /*
  583. * First get the "struct iovec" from user memory and
  584. * verify all the pointers
  585. */
  586. if (nr_segs > UIO_MAXIOV) {
  587. ret = -EINVAL;
  588. goto out;
  589. }
  590. if (nr_segs > fast_segs) {
  591. iov = kmalloc(nr_segs*sizeof(struct iovec), GFP_KERNEL);
  592. if (iov == NULL) {
  593. ret = -ENOMEM;
  594. goto out;
  595. }
  596. }
  597. if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) {
  598. ret = -EFAULT;
  599. goto out;
  600. }
  601. /*
  602. * According to the Single Unix Specification we should return EINVAL
  603. * if an element length is < 0 when cast to ssize_t or if the
  604. * total length would overflow the ssize_t return value of the
  605. * system call.
  606. *
  607. * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the
  608. * overflow case.
  609. */
  610. ret = 0;
  611. for (seg = 0; seg < nr_segs; seg++) {
  612. void __user *buf = iov[seg].iov_base;
  613. ssize_t len = (ssize_t)iov[seg].iov_len;
  614. /* see if we we're about to use an invalid len or if
  615. * it's about to overflow ssize_t */
  616. if (len < 0) {
  617. ret = -EINVAL;
  618. goto out;
  619. }
  620. if (type >= 0
  621. && unlikely(!access_ok(vrfy_dir(type), buf, len))) {
  622. ret = -EFAULT;
  623. goto out;
  624. }
  625. if (len > MAX_RW_COUNT - ret) {
  626. len = MAX_RW_COUNT - ret;
  627. iov[seg].iov_len = len;
  628. }
  629. ret += len;
  630. }
  631. out:
  632. *ret_pointer = iov;
  633. return ret;
  634. }
  635. static ssize_t do_readv_writev(int type, struct file *file,
  636. const struct iovec __user * uvector,
  637. unsigned long nr_segs, loff_t *pos)
  638. {
  639. size_t tot_len;
  640. struct iovec iovstack[UIO_FASTIOV];
  641. struct iovec *iov = iovstack;
  642. ssize_t ret;
  643. io_fn_t fn;
  644. iov_fn_t fnv;
  645. ret = rw_copy_check_uvector(type, uvector, nr_segs,
  646. ARRAY_SIZE(iovstack), iovstack, &iov);
  647. if (ret <= 0)
  648. goto out;
  649. tot_len = ret;
  650. ret = rw_verify_area(type, file, pos, tot_len);
  651. if (ret < 0)
  652. goto out;
  653. fnv = NULL;
  654. if (type == READ) {
  655. fn = file->f_op->read;
  656. fnv = file->f_op->aio_read;
  657. } else {
  658. fn = (io_fn_t)file->f_op->write;
  659. fnv = file->f_op->aio_write;
  660. file_start_write(file);
  661. }
  662. if (fnv)
  663. ret = do_sync_readv_writev(file, iov, nr_segs, tot_len,
  664. pos, fnv);
  665. else
  666. ret = do_loop_readv_writev(file, iov, nr_segs, pos, fn);
  667. if (type != READ)
  668. file_end_write(file);
  669. out:
  670. if (iov != iovstack)
  671. kfree(iov);
  672. if ((ret + (type == READ)) > 0) {
  673. if (type == READ)
  674. fsnotify_access(file);
  675. else
  676. fsnotify_modify(file);
  677. }
  678. return ret;
  679. }
  680. ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
  681. unsigned long vlen, loff_t *pos)
  682. {
  683. if (!(file->f_mode & FMODE_READ))
  684. return -EBADF;
  685. if (!file->f_op->aio_read && !file->f_op->read)
  686. return -EINVAL;
  687. return do_readv_writev(READ, file, vec, vlen, pos);
  688. }
  689. EXPORT_SYMBOL(vfs_readv);
  690. ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
  691. unsigned long vlen, loff_t *pos)
  692. {
  693. if (!(file->f_mode & FMODE_WRITE))
  694. return -EBADF;
  695. if (!file->f_op->aio_write && !file->f_op->write)
  696. return -EINVAL;
  697. return do_readv_writev(WRITE, file, vec, vlen, pos);
  698. }
  699. EXPORT_SYMBOL(vfs_writev);
  700. SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec,
  701. unsigned long, vlen)
  702. {
  703. struct fd f = fdget(fd);
  704. ssize_t ret = -EBADF;
  705. if (f.file) {
  706. loff_t pos = file_pos_read(f.file);
  707. ret = vfs_readv(f.file, vec, vlen, &pos);
  708. if (ret >= 0)
  709. file_pos_write(f.file, pos);
  710. fdput(f);
  711. }
  712. if (ret > 0)
  713. add_rchar(current, ret);
  714. inc_syscr(current);
  715. return ret;
  716. }
  717. SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec,
  718. unsigned long, vlen)
  719. {
  720. struct fd f = fdget(fd);
  721. ssize_t ret = -EBADF;
  722. if (f.file) {
  723. loff_t pos = file_pos_read(f.file);
  724. ret = vfs_writev(f.file, vec, vlen, &pos);
  725. if (ret >= 0)
  726. file_pos_write(f.file, pos);
  727. fdput(f);
  728. }
  729. if (ret > 0)
  730. add_wchar(current, ret);
  731. inc_syscw(current);
  732. return ret;
  733. }
  734. static inline loff_t pos_from_hilo(unsigned long high, unsigned long low)
  735. {
  736. #define HALF_LONG_BITS (BITS_PER_LONG / 2)
  737. return (((loff_t)high << HALF_LONG_BITS) << HALF_LONG_BITS) | low;
  738. }
  739. SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec,
  740. unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
  741. {
  742. loff_t pos = pos_from_hilo(pos_h, pos_l);
  743. struct fd f;
  744. ssize_t ret = -EBADF;
  745. if (pos < 0)
  746. return -EINVAL;
  747. f = fdget(fd);
  748. if (f.file) {
  749. ret = -ESPIPE;
  750. if (f.file->f_mode & FMODE_PREAD)
  751. ret = vfs_readv(f.file, vec, vlen, &pos);
  752. fdput(f);
  753. }
  754. if (ret > 0)
  755. add_rchar(current, ret);
  756. inc_syscr(current);
  757. return ret;
  758. }
  759. SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec,
  760. unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
  761. {
  762. loff_t pos = pos_from_hilo(pos_h, pos_l);
  763. struct fd f;
  764. ssize_t ret = -EBADF;
  765. if (pos < 0)
  766. return -EINVAL;
  767. f = fdget(fd);
  768. if (f.file) {
  769. ret = -ESPIPE;
  770. if (f.file->f_mode & FMODE_PWRITE)
  771. ret = vfs_writev(f.file, vec, vlen, &pos);
  772. fdput(f);
  773. }
  774. if (ret > 0)
  775. add_wchar(current, ret);
  776. inc_syscw(current);
  777. return ret;
  778. }
  779. #ifdef CONFIG_COMPAT
  780. static ssize_t compat_do_readv_writev(int type, struct file *file,
  781. const struct compat_iovec __user *uvector,
  782. unsigned long nr_segs, loff_t *pos)
  783. {
  784. compat_ssize_t tot_len;
  785. struct iovec iovstack[UIO_FASTIOV];
  786. struct iovec *iov = iovstack;
  787. ssize_t ret;
  788. io_fn_t fn;
  789. iov_fn_t fnv;
  790. ret = -EFAULT;
  791. if (!access_ok(VERIFY_READ, uvector, nr_segs*sizeof(*uvector)))
  792. goto out;
  793. ret = compat_rw_copy_check_uvector(type, uvector, nr_segs,
  794. UIO_FASTIOV, iovstack, &iov);
  795. if (ret <= 0)
  796. goto out;
  797. tot_len = ret;
  798. ret = rw_verify_area(type, file, pos, tot_len);
  799. if (ret < 0)
  800. goto out;
  801. fnv = NULL;
  802. if (type == READ) {
  803. fn = file->f_op->read;
  804. fnv = file->f_op->aio_read;
  805. } else {
  806. fn = (io_fn_t)file->f_op->write;
  807. fnv = file->f_op->aio_write;
  808. file_start_write(file);
  809. }
  810. if (fnv)
  811. ret = do_sync_readv_writev(file, iov, nr_segs, tot_len,
  812. pos, fnv);
  813. else
  814. ret = do_loop_readv_writev(file, iov, nr_segs, pos, fn);
  815. if (type != READ)
  816. file_end_write(file);
  817. out:
  818. if (iov != iovstack)
  819. kfree(iov);
  820. if ((ret + (type == READ)) > 0) {
  821. if (type == READ)
  822. fsnotify_access(file);
  823. else
  824. fsnotify_modify(file);
  825. }
  826. return ret;
  827. }
  828. static size_t compat_readv(struct file *file,
  829. const struct compat_iovec __user *vec,
  830. unsigned long vlen, loff_t *pos)
  831. {
  832. ssize_t ret = -EBADF;
  833. if (!(file->f_mode & FMODE_READ))
  834. goto out;
  835. ret = -EINVAL;
  836. if (!file->f_op->aio_read && !file->f_op->read)
  837. goto out;
  838. ret = compat_do_readv_writev(READ, file, vec, vlen, pos);
  839. out:
  840. if (ret > 0)
  841. add_rchar(current, ret);
  842. inc_syscr(current);
  843. return ret;
  844. }
  845. COMPAT_SYSCALL_DEFINE3(readv, unsigned long, fd,
  846. const struct compat_iovec __user *,vec,
  847. unsigned long, vlen)
  848. {
  849. struct fd f = fdget(fd);
  850. ssize_t ret;
  851. loff_t pos;
  852. if (!f.file)
  853. return -EBADF;
  854. pos = f.file->f_pos;
  855. ret = compat_readv(f.file, vec, vlen, &pos);
  856. if (ret >= 0)
  857. f.file->f_pos = pos;
  858. fdput(f);
  859. return ret;
  860. }
  861. COMPAT_SYSCALL_DEFINE4(preadv64, unsigned long, fd,
  862. const struct compat_iovec __user *,vec,
  863. unsigned long, vlen, loff_t, pos)
  864. {
  865. struct fd f;
  866. ssize_t ret;
  867. if (pos < 0)
  868. return -EINVAL;
  869. f = fdget(fd);
  870. if (!f.file)
  871. return -EBADF;
  872. ret = -ESPIPE;
  873. if (f.file->f_mode & FMODE_PREAD)
  874. ret = compat_readv(f.file, vec, vlen, &pos);
  875. fdput(f);
  876. return ret;
  877. }
  878. COMPAT_SYSCALL_DEFINE5(preadv, unsigned long, fd,
  879. const struct compat_iovec __user *,vec,
  880. unsigned long, vlen, u32, pos_low, u32, pos_high)
  881. {
  882. loff_t pos = ((loff_t)pos_high << 32) | pos_low;
  883. return compat_sys_preadv64(fd, vec, vlen, pos);
  884. }
  885. static size_t compat_writev(struct file *file,
  886. const struct compat_iovec __user *vec,
  887. unsigned long vlen, loff_t *pos)
  888. {
  889. ssize_t ret = -EBADF;
  890. if (!(file->f_mode & FMODE_WRITE))
  891. goto out;
  892. ret = -EINVAL;
  893. if (!file->f_op->aio_write && !file->f_op->write)
  894. goto out;
  895. ret = compat_do_readv_writev(WRITE, file, vec, vlen, pos);
  896. out:
  897. if (ret > 0)
  898. add_wchar(current, ret);
  899. inc_syscw(current);
  900. return ret;
  901. }
  902. COMPAT_SYSCALL_DEFINE3(writev, unsigned long, fd,
  903. const struct compat_iovec __user *, vec,
  904. unsigned long, vlen)
  905. {
  906. struct fd f = fdget(fd);
  907. ssize_t ret;
  908. loff_t pos;
  909. if (!f.file)
  910. return -EBADF;
  911. pos = f.file->f_pos;
  912. ret = compat_writev(f.file, vec, vlen, &pos);
  913. if (ret >= 0)
  914. f.file->f_pos = pos;
  915. fdput(f);
  916. return ret;
  917. }
  918. COMPAT_SYSCALL_DEFINE4(pwritev64, unsigned long, fd,
  919. const struct compat_iovec __user *,vec,
  920. unsigned long, vlen, loff_t, pos)
  921. {
  922. struct fd f;
  923. ssize_t ret;
  924. if (pos < 0)
  925. return -EINVAL;
  926. f = fdget(fd);
  927. if (!f.file)
  928. return -EBADF;
  929. ret = -ESPIPE;
  930. if (f.file->f_mode & FMODE_PWRITE)
  931. ret = compat_writev(f.file, vec, vlen, &pos);
  932. fdput(f);
  933. return ret;
  934. }
  935. COMPAT_SYSCALL_DEFINE5(pwritev, unsigned long, fd,
  936. const struct compat_iovec __user *,vec,
  937. unsigned long, vlen, u32, pos_low, u32, pos_high)
  938. {
  939. loff_t pos = ((loff_t)pos_high << 32) | pos_low;
  940. return compat_sys_pwritev64(fd, vec, vlen, pos);
  941. }
  942. #endif
  943. static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
  944. size_t count, loff_t max)
  945. {
  946. struct fd in, out;
  947. struct inode *in_inode, *out_inode;
  948. loff_t pos;
  949. loff_t out_pos;
  950. ssize_t retval;
  951. int fl;
  952. /*
  953. * Get input file, and verify that it is ok..
  954. */
  955. retval = -EBADF;
  956. in = fdget(in_fd);
  957. if (!in.file)
  958. goto out;
  959. if (!(in.file->f_mode & FMODE_READ))
  960. goto fput_in;
  961. retval = -ESPIPE;
  962. if (!ppos) {
  963. pos = in.file->f_pos;
  964. } else {
  965. pos = *ppos;
  966. if (!(in.file->f_mode & FMODE_PREAD))
  967. goto fput_in;
  968. }
  969. retval = rw_verify_area(READ, in.file, &pos, count);
  970. if (retval < 0)
  971. goto fput_in;
  972. count = retval;
  973. /*
  974. * Get output file, and verify that it is ok..
  975. */
  976. retval = -EBADF;
  977. out = fdget(out_fd);
  978. if (!out.file)
  979. goto fput_in;
  980. if (!(out.file->f_mode & FMODE_WRITE))
  981. goto fput_out;
  982. retval = -EINVAL;
  983. in_inode = file_inode(in.file);
  984. out_inode = file_inode(out.file);
  985. out_pos = out.file->f_pos;
  986. retval = rw_verify_area(WRITE, out.file, &out_pos, count);
  987. if (retval < 0)
  988. goto fput_out;
  989. count = retval;
  990. if (!max)
  991. max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
  992. if (unlikely(pos + count > max)) {
  993. retval = -EOVERFLOW;
  994. if (pos >= max)
  995. goto fput_out;
  996. count = max - pos;
  997. }
  998. fl = 0;
  999. #if 0
  1000. /*
  1001. * We need to debate whether we can enable this or not. The
  1002. * man page documents EAGAIN return for the output at least,
  1003. * and the application is arguably buggy if it doesn't expect
  1004. * EAGAIN on a non-blocking file descriptor.
  1005. */
  1006. if (in.file->f_flags & O_NONBLOCK)
  1007. fl = SPLICE_F_NONBLOCK;
  1008. #endif
  1009. file_start_write(out.file);
  1010. retval = do_splice_direct(in.file, &pos, out.file, &out_pos, count, fl);
  1011. file_end_write(out.file);
  1012. if (retval > 0) {
  1013. add_rchar(current, retval);
  1014. add_wchar(current, retval);
  1015. fsnotify_access(in.file);
  1016. fsnotify_modify(out.file);
  1017. out.file->f_pos = out_pos;
  1018. if (ppos)
  1019. *ppos = pos;
  1020. else
  1021. in.file->f_pos = pos;
  1022. }
  1023. inc_syscr(current);
  1024. inc_syscw(current);
  1025. if (pos > max)
  1026. retval = -EOVERFLOW;
  1027. fput_out:
  1028. fdput(out);
  1029. fput_in:
  1030. fdput(in);
  1031. out:
  1032. return retval;
  1033. }
  1034. SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count)
  1035. {
  1036. loff_t pos;
  1037. off_t off;
  1038. ssize_t ret;
  1039. if (offset) {
  1040. if (unlikely(get_user(off, offset)))
  1041. return -EFAULT;
  1042. pos = off;
  1043. ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
  1044. if (unlikely(put_user(pos, offset)))
  1045. return -EFAULT;
  1046. return ret;
  1047. }
  1048. return do_sendfile(out_fd, in_fd, NULL, count, 0);
  1049. }
  1050. SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count)
  1051. {
  1052. loff_t pos;
  1053. ssize_t ret;
  1054. if (offset) {
  1055. if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
  1056. return -EFAULT;
  1057. ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
  1058. if (unlikely(put_user(pos, offset)))
  1059. return -EFAULT;
  1060. return ret;
  1061. }
  1062. return do_sendfile(out_fd, in_fd, NULL, count, 0);
  1063. }
  1064. #ifdef CONFIG_COMPAT
  1065. COMPAT_SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd,
  1066. compat_off_t __user *, offset, compat_size_t, count)
  1067. {
  1068. loff_t pos;
  1069. off_t off;
  1070. ssize_t ret;
  1071. if (offset) {
  1072. if (unlikely(get_user(off, offset)))
  1073. return -EFAULT;
  1074. pos = off;
  1075. ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
  1076. if (unlikely(put_user(pos, offset)))
  1077. return -EFAULT;
  1078. return ret;
  1079. }
  1080. return do_sendfile(out_fd, in_fd, NULL, count, 0);
  1081. }
  1082. COMPAT_SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd,
  1083. compat_loff_t __user *, offset, compat_size_t, count)
  1084. {
  1085. loff_t pos;
  1086. ssize_t ret;
  1087. if (offset) {
  1088. if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
  1089. return -EFAULT;
  1090. ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
  1091. if (unlikely(put_user(pos, offset)))
  1092. return -EFAULT;
  1093. return ret;
  1094. }
  1095. return do_sendfile(out_fd, in_fd, NULL, count, 0);
  1096. }
  1097. #endif