direct-io.c 39 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391
  1. /*
  2. * fs/direct-io.c
  3. *
  4. * Copyright (C) 2002, Linus Torvalds.
  5. *
  6. * O_DIRECT
  7. *
  8. * 04Jul2002 Andrew Morton
  9. * Initial version
  10. * 11Sep2002 janetinc@us.ibm.com
  11. * added readv/writev support.
  12. * 29Oct2002 Andrew Morton
  13. * rewrote bio_add_page() support.
  14. * 30Oct2002 pbadari@us.ibm.com
  15. * added support for non-aligned IO.
  16. * 06Nov2002 pbadari@us.ibm.com
  17. * added asynchronous IO support.
  18. * 21Jul2003 nathans@sgi.com
  19. * added IO completion notifier.
  20. */
  21. #include <linux/kernel.h>
  22. #include <linux/module.h>
  23. #include <linux/types.h>
  24. #include <linux/fs.h>
  25. #include <linux/mm.h>
  26. #include <linux/slab.h>
  27. #include <linux/highmem.h>
  28. #include <linux/pagemap.h>
  29. #include <linux/task_io_accounting_ops.h>
  30. #include <linux/bio.h>
  31. #include <linux/wait.h>
  32. #include <linux/err.h>
  33. #include <linux/blkdev.h>
  34. #include <linux/buffer_head.h>
  35. #include <linux/rwsem.h>
  36. #include <linux/uio.h>
  37. #include <linux/atomic.h>
  38. #include <linux/prefetch.h>
  39. #include <linux/aio.h>
  40. /*
  41. * How many user pages to map in one call to get_user_pages(). This determines
  42. * the size of a structure in the slab cache
  43. */
  44. #define DIO_PAGES 64
  45. /*
  46. * This code generally works in units of "dio_blocks". A dio_block is
  47. * somewhere between the hard sector size and the filesystem block size. it
  48. * is determined on a per-invocation basis. When talking to the filesystem
  49. * we need to convert dio_blocks to fs_blocks by scaling the dio_block quantity
  50. * down by dio->blkfactor. Similarly, fs-blocksize quantities are converted
  51. * to bio_block quantities by shifting left by blkfactor.
  52. *
  53. * If blkfactor is zero then the user's request was aligned to the filesystem's
  54. * blocksize.
  55. */
  56. /* dio_state only used in the submission path */
  57. struct dio_submit {
  58. struct bio *bio; /* bio under assembly */
  59. unsigned blkbits; /* doesn't change */
  60. unsigned blkfactor; /* When we're using an alignment which
  61. is finer than the filesystem's soft
  62. blocksize, this specifies how much
  63. finer. blkfactor=2 means 1/4-block
  64. alignment. Does not change */
  65. unsigned start_zero_done; /* flag: sub-blocksize zeroing has
  66. been performed at the start of a
  67. write */
  68. int pages_in_io; /* approximate total IO pages */
  69. size_t size; /* total request size (doesn't change)*/
  70. sector_t block_in_file; /* Current offset into the underlying
  71. file in dio_block units. */
  72. unsigned blocks_available; /* At block_in_file. changes */
  73. int reap_counter; /* rate limit reaping */
  74. sector_t final_block_in_request;/* doesn't change */
  75. unsigned first_block_in_page; /* doesn't change, Used only once */
  76. int boundary; /* prev block is at a boundary */
  77. get_block_t *get_block; /* block mapping function */
  78. dio_submit_t *submit_io; /* IO submition function */
  79. loff_t logical_offset_in_bio; /* current first logical block in bio */
  80. sector_t final_block_in_bio; /* current final block in bio + 1 */
  81. sector_t next_block_for_io; /* next block to be put under IO,
  82. in dio_blocks units */
  83. /*
  84. * Deferred addition of a page to the dio. These variables are
  85. * private to dio_send_cur_page(), submit_page_section() and
  86. * dio_bio_add_page().
  87. */
  88. struct page *cur_page; /* The page */
  89. unsigned cur_page_offset; /* Offset into it, in bytes */
  90. unsigned cur_page_len; /* Nr of bytes at cur_page_offset */
  91. sector_t cur_page_block; /* Where it starts */
  92. loff_t cur_page_fs_offset; /* Offset in file */
  93. /*
  94. * Page fetching state. These variables belong to dio_refill_pages().
  95. */
  96. int curr_page; /* changes */
  97. int total_pages; /* doesn't change */
  98. unsigned long curr_user_address;/* changes */
  99. /*
  100. * Page queue. These variables belong to dio_refill_pages() and
  101. * dio_get_page().
  102. */
  103. unsigned head; /* next page to process */
  104. unsigned tail; /* last valid page + 1 */
  105. };
  106. /* dio_state communicated between submission path and end_io */
  107. struct dio {
  108. int flags; /* doesn't change */
  109. int rw;
  110. struct inode *inode;
  111. loff_t i_size; /* i_size when submitted */
  112. dio_iodone_t *end_io; /* IO completion function */
  113. void *private; /* copy from map_bh.b_private */
  114. /* BIO completion state */
  115. spinlock_t bio_lock; /* protects BIO fields below */
  116. int page_errors; /* errno from get_user_pages() */
  117. int is_async; /* is IO async ? */
  118. bool defer_completion; /* defer AIO completion to workqueue? */
  119. int io_error; /* IO error in completion path */
  120. unsigned long refcount; /* direct_io_worker() and bios */
  121. struct bio *bio_list; /* singly linked via bi_private */
  122. struct task_struct *waiter; /* waiting task (NULL if none) */
  123. /* AIO related stuff */
  124. struct kiocb *iocb; /* kiocb */
  125. ssize_t result; /* IO result */
  126. /*
  127. * pages[] (and any fields placed after it) are not zeroed out at
  128. * allocation time. Don't add new fields after pages[] unless you
  129. * wish that they not be zeroed.
  130. */
  131. union {
  132. struct page *pages[DIO_PAGES]; /* page buffer */
  133. struct work_struct complete_work;/* deferred AIO completion */
  134. };
  135. } ____cacheline_aligned_in_smp;
  136. static struct kmem_cache *dio_cache __read_mostly;
  137. /*
  138. * How many pages are in the queue?
  139. */
  140. static inline unsigned dio_pages_present(struct dio_submit *sdio)
  141. {
  142. return sdio->tail - sdio->head;
  143. }
  144. /*
  145. * Go grab and pin some userspace pages. Typically we'll get 64 at a time.
  146. */
  147. static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio)
  148. {
  149. int ret;
  150. int nr_pages;
  151. nr_pages = min(sdio->total_pages - sdio->curr_page, DIO_PAGES);
  152. ret = get_user_pages_fast(
  153. sdio->curr_user_address, /* Where from? */
  154. nr_pages, /* How many pages? */
  155. dio->rw == READ, /* Write to memory? */
  156. &dio->pages[0]); /* Put results here */
  157. if (ret < 0 && sdio->blocks_available && (dio->rw & WRITE)) {
  158. struct page *page = ZERO_PAGE(0);
  159. /*
  160. * A memory fault, but the filesystem has some outstanding
  161. * mapped blocks. We need to use those blocks up to avoid
  162. * leaking stale data in the file.
  163. */
  164. if (dio->page_errors == 0)
  165. dio->page_errors = ret;
  166. page_cache_get(page);
  167. dio->pages[0] = page;
  168. sdio->head = 0;
  169. sdio->tail = 1;
  170. ret = 0;
  171. goto out;
  172. }
  173. if (ret >= 0) {
  174. sdio->curr_user_address += ret * PAGE_SIZE;
  175. sdio->curr_page += ret;
  176. sdio->head = 0;
  177. sdio->tail = ret;
  178. ret = 0;
  179. }
  180. out:
  181. return ret;
  182. }
  183. /*
  184. * Get another userspace page. Returns an ERR_PTR on error. Pages are
  185. * buffered inside the dio so that we can call get_user_pages() against a
  186. * decent number of pages, less frequently. To provide nicer use of the
  187. * L1 cache.
  188. */
  189. static inline struct page *dio_get_page(struct dio *dio,
  190. struct dio_submit *sdio)
  191. {
  192. if (dio_pages_present(sdio) == 0) {
  193. int ret;
  194. ret = dio_refill_pages(dio, sdio);
  195. if (ret)
  196. return ERR_PTR(ret);
  197. BUG_ON(dio_pages_present(sdio) == 0);
  198. }
  199. return dio->pages[sdio->head++];
  200. }
  201. /**
  202. * dio_complete() - called when all DIO BIO I/O has been completed
  203. * @offset: the byte offset in the file of the completed operation
  204. *
  205. * This drops i_dio_count, lets interested parties know that a DIO operation
  206. * has completed, and calculates the resulting return code for the operation.
  207. *
  208. * It lets the filesystem know if it registered an interest earlier via
  209. * get_block. Pass the private field of the map buffer_head so that
  210. * filesystems can use it to hold additional state between get_block calls and
  211. * dio_complete.
  212. */
  213. static ssize_t dio_complete(struct dio *dio, loff_t offset, ssize_t ret,
  214. bool is_async)
  215. {
  216. ssize_t transferred = 0;
  217. /*
  218. * AIO submission can race with bio completion to get here while
  219. * expecting to have the last io completed by bio completion.
  220. * In that case -EIOCBQUEUED is in fact not an error we want
  221. * to preserve through this call.
  222. */
  223. if (ret == -EIOCBQUEUED)
  224. ret = 0;
  225. if (dio->result) {
  226. transferred = dio->result;
  227. /* Check for short read case */
  228. if ((dio->rw == READ) && ((offset + transferred) > dio->i_size))
  229. transferred = dio->i_size - offset;
  230. }
  231. if (ret == 0)
  232. ret = dio->page_errors;
  233. if (ret == 0)
  234. ret = dio->io_error;
  235. if (ret == 0)
  236. ret = transferred;
  237. if (dio->end_io && dio->result)
  238. dio->end_io(dio->iocb, offset, transferred, dio->private);
  239. inode_dio_done(dio->inode);
  240. if (is_async) {
  241. if (dio->rw & WRITE) {
  242. int err;
  243. err = generic_write_sync(dio->iocb->ki_filp, offset,
  244. transferred);
  245. if (err < 0 && ret > 0)
  246. ret = err;
  247. }
  248. aio_complete(dio->iocb, ret, 0);
  249. }
  250. kmem_cache_free(dio_cache, dio);
  251. return ret;
  252. }
  253. static void dio_aio_complete_work(struct work_struct *work)
  254. {
  255. struct dio *dio = container_of(work, struct dio, complete_work);
  256. dio_complete(dio, dio->iocb->ki_pos, 0, true);
  257. }
  258. static int dio_bio_complete(struct dio *dio, struct bio *bio);
  259. /*
  260. * Asynchronous IO callback.
  261. */
  262. static void dio_bio_end_aio(struct bio *bio, int error)
  263. {
  264. struct dio *dio = bio->bi_private;
  265. unsigned long remaining;
  266. unsigned long flags;
  267. /* cleanup the bio */
  268. dio_bio_complete(dio, bio);
  269. spin_lock_irqsave(&dio->bio_lock, flags);
  270. remaining = --dio->refcount;
  271. if (remaining == 1 && dio->waiter)
  272. wake_up_process(dio->waiter);
  273. spin_unlock_irqrestore(&dio->bio_lock, flags);
  274. if (remaining == 0) {
  275. if (dio->result && dio->defer_completion) {
  276. INIT_WORK(&dio->complete_work, dio_aio_complete_work);
  277. queue_work(dio->inode->i_sb->s_dio_done_wq,
  278. &dio->complete_work);
  279. } else {
  280. dio_complete(dio, dio->iocb->ki_pos, 0, true);
  281. }
  282. }
  283. }
  284. /*
  285. * The BIO completion handler simply queues the BIO up for the process-context
  286. * handler.
  287. *
  288. * During I/O bi_private points at the dio. After I/O, bi_private is used to
  289. * implement a singly-linked list of completed BIOs, at dio->bio_list.
  290. */
  291. static void dio_bio_end_io(struct bio *bio, int error)
  292. {
  293. struct dio *dio = bio->bi_private;
  294. unsigned long flags;
  295. spin_lock_irqsave(&dio->bio_lock, flags);
  296. bio->bi_private = dio->bio_list;
  297. dio->bio_list = bio;
  298. if (--dio->refcount == 1 && dio->waiter)
  299. wake_up_process(dio->waiter);
  300. spin_unlock_irqrestore(&dio->bio_lock, flags);
  301. }
  302. /**
  303. * dio_end_io - handle the end io action for the given bio
  304. * @bio: The direct io bio thats being completed
  305. * @error: Error if there was one
  306. *
  307. * This is meant to be called by any filesystem that uses their own dio_submit_t
  308. * so that the DIO specific endio actions are dealt with after the filesystem
  309. * has done it's completion work.
  310. */
  311. void dio_end_io(struct bio *bio, int error)
  312. {
  313. struct dio *dio = bio->bi_private;
  314. if (dio->is_async)
  315. dio_bio_end_aio(bio, error);
  316. else
  317. dio_bio_end_io(bio, error);
  318. }
  319. EXPORT_SYMBOL_GPL(dio_end_io);
  320. static inline void
  321. dio_bio_alloc(struct dio *dio, struct dio_submit *sdio,
  322. struct block_device *bdev,
  323. sector_t first_sector, int nr_vecs)
  324. {
  325. struct bio *bio;
  326. /*
  327. * bio_alloc() is guaranteed to return a bio when called with
  328. * __GFP_WAIT and we request a valid number of vectors.
  329. */
  330. bio = bio_alloc(GFP_KERNEL, nr_vecs);
  331. bio->bi_bdev = bdev;
  332. bio->bi_sector = first_sector;
  333. if (dio->is_async)
  334. bio->bi_end_io = dio_bio_end_aio;
  335. else
  336. bio->bi_end_io = dio_bio_end_io;
  337. sdio->bio = bio;
  338. sdio->logical_offset_in_bio = sdio->cur_page_fs_offset;
  339. }
  340. /*
  341. * In the AIO read case we speculatively dirty the pages before starting IO.
  342. * During IO completion, any of these pages which happen to have been written
  343. * back will be redirtied by bio_check_pages_dirty().
  344. *
  345. * bios hold a dio reference between submit_bio and ->end_io.
  346. */
  347. static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio)
  348. {
  349. struct bio *bio = sdio->bio;
  350. unsigned long flags;
  351. bio->bi_private = dio;
  352. spin_lock_irqsave(&dio->bio_lock, flags);
  353. dio->refcount++;
  354. spin_unlock_irqrestore(&dio->bio_lock, flags);
  355. if (dio->is_async && dio->rw == READ)
  356. bio_set_pages_dirty(bio);
  357. if (sdio->submit_io)
  358. sdio->submit_io(dio->rw, bio, dio->inode,
  359. sdio->logical_offset_in_bio);
  360. else
  361. submit_bio(dio->rw, bio);
  362. sdio->bio = NULL;
  363. sdio->boundary = 0;
  364. sdio->logical_offset_in_bio = 0;
  365. }
  366. /*
  367. * Release any resources in case of a failure
  368. */
  369. static inline void dio_cleanup(struct dio *dio, struct dio_submit *sdio)
  370. {
  371. while (dio_pages_present(sdio))
  372. page_cache_release(dio_get_page(dio, sdio));
  373. }
  374. /*
  375. * Wait for the next BIO to complete. Remove it and return it. NULL is
  376. * returned once all BIOs have been completed. This must only be called once
  377. * all bios have been issued so that dio->refcount can only decrease. This
  378. * requires that that the caller hold a reference on the dio.
  379. */
  380. static struct bio *dio_await_one(struct dio *dio)
  381. {
  382. unsigned long flags;
  383. struct bio *bio = NULL;
  384. spin_lock_irqsave(&dio->bio_lock, flags);
  385. /*
  386. * Wait as long as the list is empty and there are bios in flight. bio
  387. * completion drops the count, maybe adds to the list, and wakes while
  388. * holding the bio_lock so we don't need set_current_state()'s barrier
  389. * and can call it after testing our condition.
  390. */
  391. while (dio->refcount > 1 && dio->bio_list == NULL) {
  392. __set_current_state(TASK_UNINTERRUPTIBLE);
  393. dio->waiter = current;
  394. spin_unlock_irqrestore(&dio->bio_lock, flags);
  395. io_schedule();
  396. /* wake up sets us TASK_RUNNING */
  397. spin_lock_irqsave(&dio->bio_lock, flags);
  398. dio->waiter = NULL;
  399. }
  400. if (dio->bio_list) {
  401. bio = dio->bio_list;
  402. dio->bio_list = bio->bi_private;
  403. }
  404. spin_unlock_irqrestore(&dio->bio_lock, flags);
  405. return bio;
  406. }
  407. /*
  408. * Process one completed BIO. No locks are held.
  409. */
  410. static int dio_bio_complete(struct dio *dio, struct bio *bio)
  411. {
  412. const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
  413. struct bio_vec *bvec;
  414. unsigned i;
  415. if (!uptodate)
  416. dio->io_error = -EIO;
  417. if (dio->is_async && dio->rw == READ) {
  418. bio_check_pages_dirty(bio); /* transfers ownership */
  419. } else {
  420. bio_for_each_segment_all(bvec, bio, i) {
  421. struct page *page = bvec->bv_page;
  422. if (dio->rw == READ && !PageCompound(page))
  423. set_page_dirty_lock(page);
  424. page_cache_release(page);
  425. }
  426. bio_put(bio);
  427. }
  428. return uptodate ? 0 : -EIO;
  429. }
  430. /*
  431. * Wait on and process all in-flight BIOs. This must only be called once
  432. * all bios have been issued so that the refcount can only decrease.
  433. * This just waits for all bios to make it through dio_bio_complete. IO
  434. * errors are propagated through dio->io_error and should be propagated via
  435. * dio_complete().
  436. */
  437. static void dio_await_completion(struct dio *dio)
  438. {
  439. struct bio *bio;
  440. do {
  441. bio = dio_await_one(dio);
  442. if (bio)
  443. dio_bio_complete(dio, bio);
  444. } while (bio);
  445. }
  446. /*
  447. * A really large O_DIRECT read or write can generate a lot of BIOs. So
  448. * to keep the memory consumption sane we periodically reap any completed BIOs
  449. * during the BIO generation phase.
  450. *
  451. * This also helps to limit the peak amount of pinned userspace memory.
  452. */
  453. static inline int dio_bio_reap(struct dio *dio, struct dio_submit *sdio)
  454. {
  455. int ret = 0;
  456. if (sdio->reap_counter++ >= 64) {
  457. while (dio->bio_list) {
  458. unsigned long flags;
  459. struct bio *bio;
  460. int ret2;
  461. spin_lock_irqsave(&dio->bio_lock, flags);
  462. bio = dio->bio_list;
  463. dio->bio_list = bio->bi_private;
  464. spin_unlock_irqrestore(&dio->bio_lock, flags);
  465. ret2 = dio_bio_complete(dio, bio);
  466. if (ret == 0)
  467. ret = ret2;
  468. }
  469. sdio->reap_counter = 0;
  470. }
  471. return ret;
  472. }
  473. /*
  474. * Create workqueue for deferred direct IO completions. We allocate the
  475. * workqueue when it's first needed. This avoids creating workqueue for
  476. * filesystems that don't need it and also allows us to create the workqueue
  477. * late enough so the we can include s_id in the name of the workqueue.
  478. */
  479. static int sb_init_dio_done_wq(struct super_block *sb)
  480. {
  481. struct workqueue_struct *old;
  482. struct workqueue_struct *wq = alloc_workqueue("dio/%s",
  483. WQ_MEM_RECLAIM, 0,
  484. sb->s_id);
  485. if (!wq)
  486. return -ENOMEM;
  487. /*
  488. * This has to be atomic as more DIOs can race to create the workqueue
  489. */
  490. old = cmpxchg(&sb->s_dio_done_wq, NULL, wq);
  491. /* Someone created workqueue before us? Free ours... */
  492. if (old)
  493. destroy_workqueue(wq);
  494. return 0;
  495. }
  496. static int dio_set_defer_completion(struct dio *dio)
  497. {
  498. struct super_block *sb = dio->inode->i_sb;
  499. if (dio->defer_completion)
  500. return 0;
  501. dio->defer_completion = true;
  502. if (!sb->s_dio_done_wq)
  503. return sb_init_dio_done_wq(sb);
  504. return 0;
  505. }
  506. /*
  507. * Call into the fs to map some more disk blocks. We record the current number
  508. * of available blocks at sdio->blocks_available. These are in units of the
  509. * fs blocksize, (1 << inode->i_blkbits).
  510. *
  511. * The fs is allowed to map lots of blocks at once. If it wants to do that,
  512. * it uses the passed inode-relative block number as the file offset, as usual.
  513. *
  514. * get_block() is passed the number of i_blkbits-sized blocks which direct_io
  515. * has remaining to do. The fs should not map more than this number of blocks.
  516. *
  517. * If the fs has mapped a lot of blocks, it should populate bh->b_size to
  518. * indicate how much contiguous disk space has been made available at
  519. * bh->b_blocknr.
  520. *
  521. * If *any* of the mapped blocks are new, then the fs must set buffer_new().
  522. * This isn't very efficient...
  523. *
  524. * In the case of filesystem holes: the fs may return an arbitrarily-large
  525. * hole by returning an appropriate value in b_size and by clearing
  526. * buffer_mapped(). However the direct-io code will only process holes one
  527. * block at a time - it will repeatedly call get_block() as it walks the hole.
  528. */
  529. static int get_more_blocks(struct dio *dio, struct dio_submit *sdio,
  530. struct buffer_head *map_bh)
  531. {
  532. int ret;
  533. sector_t fs_startblk; /* Into file, in filesystem-sized blocks */
  534. sector_t fs_endblk; /* Into file, in filesystem-sized blocks */
  535. unsigned long fs_count; /* Number of filesystem-sized blocks */
  536. int create;
  537. unsigned int i_blkbits = sdio->blkbits + sdio->blkfactor;
  538. /*
  539. * If there was a memory error and we've overwritten all the
  540. * mapped blocks then we can now return that memory error
  541. */
  542. ret = dio->page_errors;
  543. if (ret == 0) {
  544. BUG_ON(sdio->block_in_file >= sdio->final_block_in_request);
  545. fs_startblk = sdio->block_in_file >> sdio->blkfactor;
  546. fs_endblk = (sdio->final_block_in_request - 1) >>
  547. sdio->blkfactor;
  548. fs_count = fs_endblk - fs_startblk + 1;
  549. map_bh->b_state = 0;
  550. map_bh->b_size = fs_count << i_blkbits;
  551. /*
  552. * For writes inside i_size on a DIO_SKIP_HOLES filesystem we
  553. * forbid block creations: only overwrites are permitted.
  554. * We will return early to the caller once we see an
  555. * unmapped buffer head returned, and the caller will fall
  556. * back to buffered I/O.
  557. *
  558. * Otherwise the decision is left to the get_blocks method,
  559. * which may decide to handle it or also return an unmapped
  560. * buffer head.
  561. */
  562. create = dio->rw & WRITE;
  563. if (dio->flags & DIO_SKIP_HOLES) {
  564. if (sdio->block_in_file < (i_size_read(dio->inode) >>
  565. sdio->blkbits))
  566. create = 0;
  567. }
  568. ret = (*sdio->get_block)(dio->inode, fs_startblk,
  569. map_bh, create);
  570. /* Store for completion */
  571. dio->private = map_bh->b_private;
  572. if (ret == 0 && buffer_defer_completion(map_bh))
  573. ret = dio_set_defer_completion(dio);
  574. }
  575. return ret;
  576. }
  577. /*
  578. * There is no bio. Make one now.
  579. */
  580. static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio,
  581. sector_t start_sector, struct buffer_head *map_bh)
  582. {
  583. sector_t sector;
  584. int ret, nr_pages;
  585. ret = dio_bio_reap(dio, sdio);
  586. if (ret)
  587. goto out;
  588. sector = start_sector << (sdio->blkbits - 9);
  589. nr_pages = min(sdio->pages_in_io, bio_get_nr_vecs(map_bh->b_bdev));
  590. nr_pages = min(nr_pages, BIO_MAX_PAGES);
  591. BUG_ON(nr_pages <= 0);
  592. dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages);
  593. sdio->boundary = 0;
  594. out:
  595. return ret;
  596. }
  597. /*
  598. * Attempt to put the current chunk of 'cur_page' into the current BIO. If
  599. * that was successful then update final_block_in_bio and take a ref against
  600. * the just-added page.
  601. *
  602. * Return zero on success. Non-zero means the caller needs to start a new BIO.
  603. */
  604. static inline int dio_bio_add_page(struct dio_submit *sdio)
  605. {
  606. int ret;
  607. ret = bio_add_page(sdio->bio, sdio->cur_page,
  608. sdio->cur_page_len, sdio->cur_page_offset);
  609. if (ret == sdio->cur_page_len) {
  610. /*
  611. * Decrement count only, if we are done with this page
  612. */
  613. if ((sdio->cur_page_len + sdio->cur_page_offset) == PAGE_SIZE)
  614. sdio->pages_in_io--;
  615. page_cache_get(sdio->cur_page);
  616. sdio->final_block_in_bio = sdio->cur_page_block +
  617. (sdio->cur_page_len >> sdio->blkbits);
  618. ret = 0;
  619. } else {
  620. ret = 1;
  621. }
  622. return ret;
  623. }
  624. /*
  625. * Put cur_page under IO. The section of cur_page which is described by
  626. * cur_page_offset,cur_page_len is put into a BIO. The section of cur_page
  627. * starts on-disk at cur_page_block.
  628. *
  629. * We take a ref against the page here (on behalf of its presence in the bio).
  630. *
  631. * The caller of this function is responsible for removing cur_page from the
  632. * dio, and for dropping the refcount which came from that presence.
  633. */
  634. static inline int dio_send_cur_page(struct dio *dio, struct dio_submit *sdio,
  635. struct buffer_head *map_bh)
  636. {
  637. int ret = 0;
  638. if (sdio->bio) {
  639. loff_t cur_offset = sdio->cur_page_fs_offset;
  640. loff_t bio_next_offset = sdio->logical_offset_in_bio +
  641. sdio->bio->bi_size;
  642. /*
  643. * See whether this new request is contiguous with the old.
  644. *
  645. * Btrfs cannot handle having logically non-contiguous requests
  646. * submitted. For example if you have
  647. *
  648. * Logical: [0-4095][HOLE][8192-12287]
  649. * Physical: [0-4095] [4096-8191]
  650. *
  651. * We cannot submit those pages together as one BIO. So if our
  652. * current logical offset in the file does not equal what would
  653. * be the next logical offset in the bio, submit the bio we
  654. * have.
  655. */
  656. if (sdio->final_block_in_bio != sdio->cur_page_block ||
  657. cur_offset != bio_next_offset)
  658. dio_bio_submit(dio, sdio);
  659. }
  660. if (sdio->bio == NULL) {
  661. ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh);
  662. if (ret)
  663. goto out;
  664. }
  665. if (dio_bio_add_page(sdio) != 0) {
  666. dio_bio_submit(dio, sdio);
  667. ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh);
  668. if (ret == 0) {
  669. ret = dio_bio_add_page(sdio);
  670. BUG_ON(ret != 0);
  671. }
  672. }
  673. out:
  674. return ret;
  675. }
  676. /*
  677. * An autonomous function to put a chunk of a page under deferred IO.
  678. *
  679. * The caller doesn't actually know (or care) whether this piece of page is in
  680. * a BIO, or is under IO or whatever. We just take care of all possible
  681. * situations here. The separation between the logic of do_direct_IO() and
  682. * that of submit_page_section() is important for clarity. Please don't break.
  683. *
  684. * The chunk of page starts on-disk at blocknr.
  685. *
  686. * We perform deferred IO, by recording the last-submitted page inside our
  687. * private part of the dio structure. If possible, we just expand the IO
  688. * across that page here.
  689. *
  690. * If that doesn't work out then we put the old page into the bio and add this
  691. * page to the dio instead.
  692. */
  693. static inline int
  694. submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page,
  695. unsigned offset, unsigned len, sector_t blocknr,
  696. struct buffer_head *map_bh)
  697. {
  698. int ret = 0;
  699. if (dio->rw & WRITE) {
  700. /*
  701. * Read accounting is performed in submit_bio()
  702. */
  703. task_io_account_write(len);
  704. }
  705. /*
  706. * Can we just grow the current page's presence in the dio?
  707. */
  708. if (sdio->cur_page == page &&
  709. sdio->cur_page_offset + sdio->cur_page_len == offset &&
  710. sdio->cur_page_block +
  711. (sdio->cur_page_len >> sdio->blkbits) == blocknr) {
  712. sdio->cur_page_len += len;
  713. goto out;
  714. }
  715. /*
  716. * If there's a deferred page already there then send it.
  717. */
  718. if (sdio->cur_page) {
  719. ret = dio_send_cur_page(dio, sdio, map_bh);
  720. page_cache_release(sdio->cur_page);
  721. sdio->cur_page = NULL;
  722. if (ret)
  723. return ret;
  724. }
  725. page_cache_get(page); /* It is in dio */
  726. sdio->cur_page = page;
  727. sdio->cur_page_offset = offset;
  728. sdio->cur_page_len = len;
  729. sdio->cur_page_block = blocknr;
  730. sdio->cur_page_fs_offset = sdio->block_in_file << sdio->blkbits;
  731. out:
  732. /*
  733. * If sdio->boundary then we want to schedule the IO now to
  734. * avoid metadata seeks.
  735. */
  736. if (sdio->boundary) {
  737. ret = dio_send_cur_page(dio, sdio, map_bh);
  738. dio_bio_submit(dio, sdio);
  739. page_cache_release(sdio->cur_page);
  740. sdio->cur_page = NULL;
  741. }
  742. return ret;
  743. }
  744. /*
  745. * Clean any dirty buffers in the blockdev mapping which alias newly-created
  746. * file blocks. Only called for S_ISREG files - blockdevs do not set
  747. * buffer_new
  748. */
  749. static void clean_blockdev_aliases(struct dio *dio, struct buffer_head *map_bh)
  750. {
  751. unsigned i;
  752. unsigned nblocks;
  753. nblocks = map_bh->b_size >> dio->inode->i_blkbits;
  754. for (i = 0; i < nblocks; i++) {
  755. unmap_underlying_metadata(map_bh->b_bdev,
  756. map_bh->b_blocknr + i);
  757. }
  758. }
  759. /*
  760. * If we are not writing the entire block and get_block() allocated
  761. * the block for us, we need to fill-in the unused portion of the
  762. * block with zeros. This happens only if user-buffer, fileoffset or
  763. * io length is not filesystem block-size multiple.
  764. *
  765. * `end' is zero if we're doing the start of the IO, 1 at the end of the
  766. * IO.
  767. */
  768. static inline void dio_zero_block(struct dio *dio, struct dio_submit *sdio,
  769. int end, struct buffer_head *map_bh)
  770. {
  771. unsigned dio_blocks_per_fs_block;
  772. unsigned this_chunk_blocks; /* In dio_blocks */
  773. unsigned this_chunk_bytes;
  774. struct page *page;
  775. sdio->start_zero_done = 1;
  776. if (!sdio->blkfactor || !buffer_new(map_bh))
  777. return;
  778. dio_blocks_per_fs_block = 1 << sdio->blkfactor;
  779. this_chunk_blocks = sdio->block_in_file & (dio_blocks_per_fs_block - 1);
  780. if (!this_chunk_blocks)
  781. return;
  782. /*
  783. * We need to zero out part of an fs block. It is either at the
  784. * beginning or the end of the fs block.
  785. */
  786. if (end)
  787. this_chunk_blocks = dio_blocks_per_fs_block - this_chunk_blocks;
  788. this_chunk_bytes = this_chunk_blocks << sdio->blkbits;
  789. page = ZERO_PAGE(0);
  790. if (submit_page_section(dio, sdio, page, 0, this_chunk_bytes,
  791. sdio->next_block_for_io, map_bh))
  792. return;
  793. sdio->next_block_for_io += this_chunk_blocks;
  794. }
  795. /*
  796. * Walk the user pages, and the file, mapping blocks to disk and generating
  797. * a sequence of (page,offset,len,block) mappings. These mappings are injected
  798. * into submit_page_section(), which takes care of the next stage of submission
  799. *
  800. * Direct IO against a blockdev is different from a file. Because we can
  801. * happily perform page-sized but 512-byte aligned IOs. It is important that
  802. * blockdev IO be able to have fine alignment and large sizes.
  803. *
  804. * So what we do is to permit the ->get_block function to populate bh.b_size
  805. * with the size of IO which is permitted at this offset and this i_blkbits.
  806. *
  807. * For best results, the blockdev should be set up with 512-byte i_blkbits and
  808. * it should set b_size to PAGE_SIZE or more inside get_block(). This gives
  809. * fine alignment but still allows this function to work in PAGE_SIZE units.
  810. */
  811. static int do_direct_IO(struct dio *dio, struct dio_submit *sdio,
  812. struct buffer_head *map_bh)
  813. {
  814. const unsigned blkbits = sdio->blkbits;
  815. const unsigned blocks_per_page = PAGE_SIZE >> blkbits;
  816. struct page *page;
  817. unsigned block_in_page;
  818. int ret = 0;
  819. /* The I/O can start at any block offset within the first page */
  820. block_in_page = sdio->first_block_in_page;
  821. while (sdio->block_in_file < sdio->final_block_in_request) {
  822. page = dio_get_page(dio, sdio);
  823. if (IS_ERR(page)) {
  824. ret = PTR_ERR(page);
  825. goto out;
  826. }
  827. while (block_in_page < blocks_per_page) {
  828. unsigned offset_in_page = block_in_page << blkbits;
  829. unsigned this_chunk_bytes; /* # of bytes mapped */
  830. unsigned this_chunk_blocks; /* # of blocks */
  831. unsigned u;
  832. if (sdio->blocks_available == 0) {
  833. /*
  834. * Need to go and map some more disk
  835. */
  836. unsigned long blkmask;
  837. unsigned long dio_remainder;
  838. ret = get_more_blocks(dio, sdio, map_bh);
  839. if (ret) {
  840. page_cache_release(page);
  841. goto out;
  842. }
  843. if (!buffer_mapped(map_bh))
  844. goto do_holes;
  845. sdio->blocks_available =
  846. map_bh->b_size >> sdio->blkbits;
  847. sdio->next_block_for_io =
  848. map_bh->b_blocknr << sdio->blkfactor;
  849. if (buffer_new(map_bh))
  850. clean_blockdev_aliases(dio, map_bh);
  851. if (!sdio->blkfactor)
  852. goto do_holes;
  853. blkmask = (1 << sdio->blkfactor) - 1;
  854. dio_remainder = (sdio->block_in_file & blkmask);
  855. /*
  856. * If we are at the start of IO and that IO
  857. * starts partway into a fs-block,
  858. * dio_remainder will be non-zero. If the IO
  859. * is a read then we can simply advance the IO
  860. * cursor to the first block which is to be
  861. * read. But if the IO is a write and the
  862. * block was newly allocated we cannot do that;
  863. * the start of the fs block must be zeroed out
  864. * on-disk
  865. */
  866. if (!buffer_new(map_bh))
  867. sdio->next_block_for_io += dio_remainder;
  868. sdio->blocks_available -= dio_remainder;
  869. }
  870. do_holes:
  871. /* Handle holes */
  872. if (!buffer_mapped(map_bh)) {
  873. loff_t i_size_aligned;
  874. /* AKPM: eargh, -ENOTBLK is a hack */
  875. if (dio->rw & WRITE) {
  876. page_cache_release(page);
  877. return -ENOTBLK;
  878. }
  879. /*
  880. * Be sure to account for a partial block as the
  881. * last block in the file
  882. */
  883. i_size_aligned = ALIGN(i_size_read(dio->inode),
  884. 1 << blkbits);
  885. if (sdio->block_in_file >=
  886. i_size_aligned >> blkbits) {
  887. /* We hit eof */
  888. page_cache_release(page);
  889. goto out;
  890. }
  891. zero_user(page, block_in_page << blkbits,
  892. 1 << blkbits);
  893. sdio->block_in_file++;
  894. block_in_page++;
  895. goto next_block;
  896. }
  897. /*
  898. * If we're performing IO which has an alignment which
  899. * is finer than the underlying fs, go check to see if
  900. * we must zero out the start of this block.
  901. */
  902. if (unlikely(sdio->blkfactor && !sdio->start_zero_done))
  903. dio_zero_block(dio, sdio, 0, map_bh);
  904. /*
  905. * Work out, in this_chunk_blocks, how much disk we
  906. * can add to this page
  907. */
  908. this_chunk_blocks = sdio->blocks_available;
  909. u = (PAGE_SIZE - offset_in_page) >> blkbits;
  910. if (this_chunk_blocks > u)
  911. this_chunk_blocks = u;
  912. u = sdio->final_block_in_request - sdio->block_in_file;
  913. if (this_chunk_blocks > u)
  914. this_chunk_blocks = u;
  915. this_chunk_bytes = this_chunk_blocks << blkbits;
  916. BUG_ON(this_chunk_bytes == 0);
  917. if (this_chunk_blocks == sdio->blocks_available)
  918. sdio->boundary = buffer_boundary(map_bh);
  919. ret = submit_page_section(dio, sdio, page,
  920. offset_in_page,
  921. this_chunk_bytes,
  922. sdio->next_block_for_io,
  923. map_bh);
  924. if (ret) {
  925. page_cache_release(page);
  926. goto out;
  927. }
  928. sdio->next_block_for_io += this_chunk_blocks;
  929. sdio->block_in_file += this_chunk_blocks;
  930. block_in_page += this_chunk_blocks;
  931. sdio->blocks_available -= this_chunk_blocks;
  932. next_block:
  933. BUG_ON(sdio->block_in_file > sdio->final_block_in_request);
  934. if (sdio->block_in_file == sdio->final_block_in_request)
  935. break;
  936. }
  937. /* Drop the ref which was taken in get_user_pages() */
  938. page_cache_release(page);
  939. block_in_page = 0;
  940. }
  941. out:
  942. return ret;
  943. }
  944. static inline int drop_refcount(struct dio *dio)
  945. {
  946. int ret2;
  947. unsigned long flags;
  948. /*
  949. * Sync will always be dropping the final ref and completing the
  950. * operation. AIO can if it was a broken operation described above or
  951. * in fact if all the bios race to complete before we get here. In
  952. * that case dio_complete() translates the EIOCBQUEUED into the proper
  953. * return code that the caller will hand to aio_complete().
  954. *
  955. * This is managed by the bio_lock instead of being an atomic_t so that
  956. * completion paths can drop their ref and use the remaining count to
  957. * decide to wake the submission path atomically.
  958. */
  959. spin_lock_irqsave(&dio->bio_lock, flags);
  960. ret2 = --dio->refcount;
  961. spin_unlock_irqrestore(&dio->bio_lock, flags);
  962. return ret2;
  963. }
  964. /*
  965. * This is a library function for use by filesystem drivers.
  966. *
  967. * The locking rules are governed by the flags parameter:
  968. * - if the flags value contains DIO_LOCKING we use a fancy locking
  969. * scheme for dumb filesystems.
  970. * For writes this function is called under i_mutex and returns with
  971. * i_mutex held, for reads, i_mutex is not held on entry, but it is
  972. * taken and dropped again before returning.
  973. * - if the flags value does NOT contain DIO_LOCKING we don't use any
  974. * internal locking but rather rely on the filesystem to synchronize
  975. * direct I/O reads/writes versus each other and truncate.
  976. *
  977. * To help with locking against truncate we incremented the i_dio_count
  978. * counter before starting direct I/O, and decrement it once we are done.
  979. * Truncate can wait for it to reach zero to provide exclusion. It is
  980. * expected that filesystem provide exclusion between new direct I/O
  981. * and truncates. For DIO_LOCKING filesystems this is done by i_mutex,
  982. * but other filesystems need to take care of this on their own.
  983. *
  984. * NOTE: if you pass "sdio" to anything by pointer make sure that function
  985. * is always inlined. Otherwise gcc is unable to split the structure into
  986. * individual fields and will generate much worse code. This is important
  987. * for the whole file.
  988. */
  989. static inline ssize_t
  990. do_blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode,
  991. struct block_device *bdev, const struct iovec *iov, loff_t offset,
  992. unsigned long nr_segs, get_block_t get_block, dio_iodone_t end_io,
  993. dio_submit_t submit_io, int flags)
  994. {
  995. int seg;
  996. size_t size;
  997. unsigned long addr;
  998. unsigned i_blkbits = ACCESS_ONCE(inode->i_blkbits);
  999. unsigned blkbits = i_blkbits;
  1000. unsigned blocksize_mask = (1 << blkbits) - 1;
  1001. ssize_t retval = -EINVAL;
  1002. loff_t end = offset;
  1003. struct dio *dio;
  1004. struct dio_submit sdio = { 0, };
  1005. unsigned long user_addr;
  1006. size_t bytes;
  1007. struct buffer_head map_bh = { 0, };
  1008. struct blk_plug plug;
  1009. if (rw & WRITE)
  1010. rw = WRITE_ODIRECT;
  1011. /*
  1012. * Avoid references to bdev if not absolutely needed to give
  1013. * the early prefetch in the caller enough time.
  1014. */
  1015. if (offset & blocksize_mask) {
  1016. if (bdev)
  1017. blkbits = blksize_bits(bdev_logical_block_size(bdev));
  1018. blocksize_mask = (1 << blkbits) - 1;
  1019. if (offset & blocksize_mask)
  1020. goto out;
  1021. }
  1022. /* Check the memory alignment. Blocks cannot straddle pages */
  1023. for (seg = 0; seg < nr_segs; seg++) {
  1024. addr = (unsigned long)iov[seg].iov_base;
  1025. size = iov[seg].iov_len;
  1026. end += size;
  1027. if (unlikely((addr & blocksize_mask) ||
  1028. (size & blocksize_mask))) {
  1029. if (bdev)
  1030. blkbits = blksize_bits(
  1031. bdev_logical_block_size(bdev));
  1032. blocksize_mask = (1 << blkbits) - 1;
  1033. if ((addr & blocksize_mask) || (size & blocksize_mask))
  1034. goto out;
  1035. }
  1036. }
  1037. /* watch out for a 0 len io from a tricksy fs */
  1038. if (rw == READ && end == offset)
  1039. return 0;
  1040. dio = kmem_cache_alloc(dio_cache, GFP_KERNEL);
  1041. retval = -ENOMEM;
  1042. if (!dio)
  1043. goto out;
  1044. /*
  1045. * Believe it or not, zeroing out the page array caused a .5%
  1046. * performance regression in a database benchmark. So, we take
  1047. * care to only zero out what's needed.
  1048. */
  1049. memset(dio, 0, offsetof(struct dio, pages));
  1050. dio->flags = flags;
  1051. if (dio->flags & DIO_LOCKING) {
  1052. if (rw == READ) {
  1053. struct address_space *mapping =
  1054. iocb->ki_filp->f_mapping;
  1055. /* will be released by direct_io_worker */
  1056. mutex_lock(&inode->i_mutex);
  1057. retval = filemap_write_and_wait_range(mapping, offset,
  1058. end - 1);
  1059. if (retval) {
  1060. mutex_unlock(&inode->i_mutex);
  1061. kmem_cache_free(dio_cache, dio);
  1062. goto out;
  1063. }
  1064. }
  1065. }
  1066. /*
  1067. * For file extending writes updating i_size before data
  1068. * writeouts complete can expose uninitialized blocks. So
  1069. * even for AIO, we need to wait for i/o to complete before
  1070. * returning in this case.
  1071. */
  1072. dio->is_async = !is_sync_kiocb(iocb) && !((rw & WRITE) &&
  1073. (end > i_size_read(inode)));
  1074. dio->inode = inode;
  1075. dio->rw = rw;
  1076. /*
  1077. * For AIO O_(D)SYNC writes we need to defer completions to a workqueue
  1078. * so that we can call ->fsync.
  1079. */
  1080. if (dio->is_async && (rw & WRITE) &&
  1081. ((iocb->ki_filp->f_flags & O_DSYNC) ||
  1082. IS_SYNC(iocb->ki_filp->f_mapping->host))) {
  1083. retval = dio_set_defer_completion(dio);
  1084. if (retval) {
  1085. /*
  1086. * We grab i_mutex only for reads so we don't have
  1087. * to release it here
  1088. */
  1089. kmem_cache_free(dio_cache, dio);
  1090. goto out;
  1091. }
  1092. }
  1093. /*
  1094. * Will be decremented at I/O completion time.
  1095. */
  1096. atomic_inc(&inode->i_dio_count);
  1097. retval = 0;
  1098. sdio.blkbits = blkbits;
  1099. sdio.blkfactor = i_blkbits - blkbits;
  1100. sdio.block_in_file = offset >> blkbits;
  1101. sdio.get_block = get_block;
  1102. dio->end_io = end_io;
  1103. sdio.submit_io = submit_io;
  1104. sdio.final_block_in_bio = -1;
  1105. sdio.next_block_for_io = -1;
  1106. dio->iocb = iocb;
  1107. dio->i_size = i_size_read(inode);
  1108. spin_lock_init(&dio->bio_lock);
  1109. dio->refcount = 1;
  1110. /*
  1111. * In case of non-aligned buffers, we may need 2 more
  1112. * pages since we need to zero out first and last block.
  1113. */
  1114. if (unlikely(sdio.blkfactor))
  1115. sdio.pages_in_io = 2;
  1116. for (seg = 0; seg < nr_segs; seg++) {
  1117. user_addr = (unsigned long)iov[seg].iov_base;
  1118. sdio.pages_in_io +=
  1119. ((user_addr + iov[seg].iov_len + PAGE_SIZE-1) /
  1120. PAGE_SIZE - user_addr / PAGE_SIZE);
  1121. }
  1122. blk_start_plug(&plug);
  1123. for (seg = 0; seg < nr_segs; seg++) {
  1124. user_addr = (unsigned long)iov[seg].iov_base;
  1125. sdio.size += bytes = iov[seg].iov_len;
  1126. /* Index into the first page of the first block */
  1127. sdio.first_block_in_page = (user_addr & ~PAGE_MASK) >> blkbits;
  1128. sdio.final_block_in_request = sdio.block_in_file +
  1129. (bytes >> blkbits);
  1130. /* Page fetching state */
  1131. sdio.head = 0;
  1132. sdio.tail = 0;
  1133. sdio.curr_page = 0;
  1134. sdio.total_pages = 0;
  1135. if (user_addr & (PAGE_SIZE-1)) {
  1136. sdio.total_pages++;
  1137. bytes -= PAGE_SIZE - (user_addr & (PAGE_SIZE - 1));
  1138. }
  1139. sdio.total_pages += (bytes + PAGE_SIZE - 1) / PAGE_SIZE;
  1140. sdio.curr_user_address = user_addr;
  1141. retval = do_direct_IO(dio, &sdio, &map_bh);
  1142. dio->result += iov[seg].iov_len -
  1143. ((sdio.final_block_in_request - sdio.block_in_file) <<
  1144. blkbits);
  1145. if (retval) {
  1146. dio_cleanup(dio, &sdio);
  1147. break;
  1148. }
  1149. } /* end iovec loop */
  1150. if (retval == -ENOTBLK) {
  1151. /*
  1152. * The remaining part of the request will be
  1153. * be handled by buffered I/O when we return
  1154. */
  1155. retval = 0;
  1156. }
  1157. /*
  1158. * There may be some unwritten disk at the end of a part-written
  1159. * fs-block-sized block. Go zero that now.
  1160. */
  1161. dio_zero_block(dio, &sdio, 1, &map_bh);
  1162. if (sdio.cur_page) {
  1163. ssize_t ret2;
  1164. ret2 = dio_send_cur_page(dio, &sdio, &map_bh);
  1165. if (retval == 0)
  1166. retval = ret2;
  1167. page_cache_release(sdio.cur_page);
  1168. sdio.cur_page = NULL;
  1169. }
  1170. if (sdio.bio)
  1171. dio_bio_submit(dio, &sdio);
  1172. blk_finish_plug(&plug);
  1173. /*
  1174. * It is possible that, we return short IO due to end of file.
  1175. * In that case, we need to release all the pages we got hold on.
  1176. */
  1177. dio_cleanup(dio, &sdio);
  1178. /*
  1179. * All block lookups have been performed. For READ requests
  1180. * we can let i_mutex go now that its achieved its purpose
  1181. * of protecting us from looking up uninitialized blocks.
  1182. */
  1183. if (rw == READ && (dio->flags & DIO_LOCKING))
  1184. mutex_unlock(&dio->inode->i_mutex);
  1185. /*
  1186. * The only time we want to leave bios in flight is when a successful
  1187. * partial aio read or full aio write have been setup. In that case
  1188. * bio completion will call aio_complete. The only time it's safe to
  1189. * call aio_complete is when we return -EIOCBQUEUED, so we key on that.
  1190. * This had *better* be the only place that raises -EIOCBQUEUED.
  1191. */
  1192. BUG_ON(retval == -EIOCBQUEUED);
  1193. if (dio->is_async && retval == 0 && dio->result &&
  1194. ((rw == READ) || (dio->result == sdio.size)))
  1195. retval = -EIOCBQUEUED;
  1196. if (retval != -EIOCBQUEUED)
  1197. dio_await_completion(dio);
  1198. if (drop_refcount(dio) == 0) {
  1199. retval = dio_complete(dio, offset, retval, false);
  1200. } else
  1201. BUG_ON(retval != -EIOCBQUEUED);
  1202. out:
  1203. return retval;
  1204. }
  1205. ssize_t
  1206. __blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode,
  1207. struct block_device *bdev, const struct iovec *iov, loff_t offset,
  1208. unsigned long nr_segs, get_block_t get_block, dio_iodone_t end_io,
  1209. dio_submit_t submit_io, int flags)
  1210. {
  1211. /*
  1212. * The block device state is needed in the end to finally
  1213. * submit everything. Since it's likely to be cache cold
  1214. * prefetch it here as first thing to hide some of the
  1215. * latency.
  1216. *
  1217. * Attempt to prefetch the pieces we likely need later.
  1218. */
  1219. prefetch(&bdev->bd_disk->part_tbl);
  1220. prefetch(bdev->bd_queue);
  1221. prefetch((char *)bdev->bd_queue + SMP_CACHE_BYTES);
  1222. return do_blockdev_direct_IO(rw, iocb, inode, bdev, iov, offset,
  1223. nr_segs, get_block, end_io,
  1224. submit_io, flags);
  1225. }
  1226. EXPORT_SYMBOL(__blockdev_direct_IO);
  1227. static __init int dio_init(void)
  1228. {
  1229. dio_cache = KMEM_CACHE(dio, SLAB_PANIC);
  1230. return 0;
  1231. }
  1232. module_init(dio_init)