filemap.c 62 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444
  1. /*
  2. * linux/mm/filemap.c
  3. *
  4. * Copyright (C) 1994-1999 Linus Torvalds
  5. */
  6. /*
  7. * This file handles the generic file mmap semantics used by
  8. * most "normal" filesystems (but you don't /have/ to use this:
  9. * the NFS filesystem used to do this differently, for example)
  10. */
  11. #include <linux/config.h>
  12. #include <linux/module.h>
  13. #include <linux/slab.h>
  14. #include <linux/compiler.h>
  15. #include <linux/fs.h>
  16. #include <linux/uaccess.h>
  17. #include <linux/aio.h>
  18. #include <linux/capability.h>
  19. #include <linux/kernel_stat.h>
  20. #include <linux/mm.h>
  21. #include <linux/swap.h>
  22. #include <linux/mman.h>
  23. #include <linux/pagemap.h>
  24. #include <linux/file.h>
  25. #include <linux/uio.h>
  26. #include <linux/hash.h>
  27. #include <linux/writeback.h>
  28. #include <linux/pagevec.h>
  29. #include <linux/blkdev.h>
  30. #include <linux/security.h>
  31. #include <linux/syscalls.h>
  32. #include <linux/cpuset.h>
  33. #include "filemap.h"
  34. #include "internal.h"
  35. /*
  36. * FIXME: remove all knowledge of the buffer layer from the core VM
  37. */
  38. #include <linux/buffer_head.h> /* for generic_osync_inode */
  39. #include <asm/mman.h>
  40. static ssize_t
  41. generic_file_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
  42. loff_t offset, unsigned long nr_segs);
  43. /*
  44. * Shared mappings implemented 30.11.1994. It's not fully working yet,
  45. * though.
  46. *
  47. * Shared mappings now work. 15.8.1995 Bruno.
  48. *
  49. * finished 'unifying' the page and buffer cache and SMP-threaded the
  50. * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
  51. *
  52. * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
  53. */
  54. /*
  55. * Lock ordering:
  56. *
  57. * ->i_mmap_lock (vmtruncate)
  58. * ->private_lock (__free_pte->__set_page_dirty_buffers)
  59. * ->swap_lock (exclusive_swap_page, others)
  60. * ->mapping->tree_lock
  61. *
  62. * ->i_mutex
  63. * ->i_mmap_lock (truncate->unmap_mapping_range)
  64. *
  65. * ->mmap_sem
  66. * ->i_mmap_lock
  67. * ->page_table_lock or pte_lock (various, mainly in memory.c)
  68. * ->mapping->tree_lock (arch-dependent flush_dcache_mmap_lock)
  69. *
  70. * ->mmap_sem
  71. * ->lock_page (access_process_vm)
  72. *
  73. * ->mmap_sem
  74. * ->i_mutex (msync)
  75. *
  76. * ->i_mutex
  77. * ->i_alloc_sem (various)
  78. *
  79. * ->inode_lock
  80. * ->sb_lock (fs/fs-writeback.c)
  81. * ->mapping->tree_lock (__sync_single_inode)
  82. *
  83. * ->i_mmap_lock
  84. * ->anon_vma.lock (vma_adjust)
  85. *
  86. * ->anon_vma.lock
  87. * ->page_table_lock or pte_lock (anon_vma_prepare and various)
  88. *
  89. * ->page_table_lock or pte_lock
  90. * ->swap_lock (try_to_unmap_one)
  91. * ->private_lock (try_to_unmap_one)
  92. * ->tree_lock (try_to_unmap_one)
  93. * ->zone.lru_lock (follow_page->mark_page_accessed)
  94. * ->zone.lru_lock (check_pte_range->isolate_lru_page)
  95. * ->private_lock (page_remove_rmap->set_page_dirty)
  96. * ->tree_lock (page_remove_rmap->set_page_dirty)
  97. * ->inode_lock (page_remove_rmap->set_page_dirty)
  98. * ->inode_lock (zap_pte_range->set_page_dirty)
  99. * ->private_lock (zap_pte_range->__set_page_dirty_buffers)
  100. *
  101. * ->task->proc_lock
  102. * ->dcache_lock (proc_pid_lookup)
  103. */
  104. /*
  105. * Remove a page from the page cache and free it. Caller has to make
  106. * sure the page is locked and that nobody else uses it - or that usage
  107. * is safe. The caller must hold a write_lock on the mapping's tree_lock.
  108. */
  109. void __remove_from_page_cache(struct page *page)
  110. {
  111. struct address_space *mapping = page->mapping;
  112. radix_tree_delete(&mapping->page_tree, page->index);
  113. page->mapping = NULL;
  114. mapping->nrpages--;
  115. pagecache_acct(-1);
  116. }
  117. void remove_from_page_cache(struct page *page)
  118. {
  119. struct address_space *mapping = page->mapping;
  120. BUG_ON(!PageLocked(page));
  121. write_lock_irq(&mapping->tree_lock);
  122. __remove_from_page_cache(page);
  123. write_unlock_irq(&mapping->tree_lock);
  124. }
  125. static int sync_page(void *word)
  126. {
  127. struct address_space *mapping;
  128. struct page *page;
  129. page = container_of((unsigned long *)word, struct page, flags);
  130. /*
  131. * page_mapping() is being called without PG_locked held.
  132. * Some knowledge of the state and use of the page is used to
  133. * reduce the requirements down to a memory barrier.
  134. * The danger here is of a stale page_mapping() return value
  135. * indicating a struct address_space different from the one it's
  136. * associated with when it is associated with one.
  137. * After smp_mb(), it's either the correct page_mapping() for
  138. * the page, or an old page_mapping() and the page's own
  139. * page_mapping() has gone NULL.
  140. * The ->sync_page() address_space operation must tolerate
  141. * page_mapping() going NULL. By an amazing coincidence,
  142. * this comes about because none of the users of the page
  143. * in the ->sync_page() methods make essential use of the
  144. * page_mapping(), merely passing the page down to the backing
  145. * device's unplug functions when it's non-NULL, which in turn
  146. * ignore it for all cases but swap, where only page_private(page) is
  147. * of interest. When page_mapping() does go NULL, the entire
  148. * call stack gracefully ignores the page and returns.
  149. * -- wli
  150. */
  151. smp_mb();
  152. mapping = page_mapping(page);
  153. if (mapping && mapping->a_ops && mapping->a_ops->sync_page)
  154. mapping->a_ops->sync_page(page);
  155. io_schedule();
  156. return 0;
  157. }
  158. /**
  159. * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
  160. * @mapping: address space structure to write
  161. * @start: offset in bytes where the range starts
  162. * @end: offset in bytes where the range ends (inclusive)
  163. * @sync_mode: enable synchronous operation
  164. *
  165. * Start writeback against all of a mapping's dirty pages that lie
  166. * within the byte offsets <start, end> inclusive.
  167. *
  168. * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
  169. * opposed to a regular memory cleansing writeback. The difference between
  170. * these two operations is that if a dirty page/buffer is encountered, it must
  171. * be waited upon, and not just skipped over.
  172. */
  173. int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
  174. loff_t end, int sync_mode)
  175. {
  176. int ret;
  177. struct writeback_control wbc = {
  178. .sync_mode = sync_mode,
  179. .nr_to_write = mapping->nrpages * 2,
  180. .range_start = start,
  181. .range_end = end,
  182. };
  183. if (!mapping_cap_writeback_dirty(mapping))
  184. return 0;
  185. ret = do_writepages(mapping, &wbc);
  186. return ret;
  187. }
  188. static inline int __filemap_fdatawrite(struct address_space *mapping,
  189. int sync_mode)
  190. {
  191. return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);
  192. }
  193. int filemap_fdatawrite(struct address_space *mapping)
  194. {
  195. return __filemap_fdatawrite(mapping, WB_SYNC_ALL);
  196. }
  197. EXPORT_SYMBOL(filemap_fdatawrite);
  198. static int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
  199. loff_t end)
  200. {
  201. return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);
  202. }
  203. /**
  204. * filemap_flush - mostly a non-blocking flush
  205. * @mapping: target address_space
  206. *
  207. * This is a mostly non-blocking flush. Not suitable for data-integrity
  208. * purposes - I/O may not be started against all dirty pages.
  209. */
  210. int filemap_flush(struct address_space *mapping)
  211. {
  212. return __filemap_fdatawrite(mapping, WB_SYNC_NONE);
  213. }
  214. EXPORT_SYMBOL(filemap_flush);
  215. /**
  216. * wait_on_page_writeback_range - wait for writeback to complete
  217. * @mapping: target address_space
  218. * @start: beginning page index
  219. * @end: ending page index
  220. *
  221. * Wait for writeback to complete against pages indexed by start->end
  222. * inclusive
  223. */
  224. int wait_on_page_writeback_range(struct address_space *mapping,
  225. pgoff_t start, pgoff_t end)
  226. {
  227. struct pagevec pvec;
  228. int nr_pages;
  229. int ret = 0;
  230. pgoff_t index;
  231. if (end < start)
  232. return 0;
  233. pagevec_init(&pvec, 0);
  234. index = start;
  235. while ((index <= end) &&
  236. (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
  237. PAGECACHE_TAG_WRITEBACK,
  238. min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1)) != 0) {
  239. unsigned i;
  240. for (i = 0; i < nr_pages; i++) {
  241. struct page *page = pvec.pages[i];
  242. /* until radix tree lookup accepts end_index */
  243. if (page->index > end)
  244. continue;
  245. wait_on_page_writeback(page);
  246. if (PageError(page))
  247. ret = -EIO;
  248. }
  249. pagevec_release(&pvec);
  250. cond_resched();
  251. }
  252. /* Check for outstanding write errors */
  253. if (test_and_clear_bit(AS_ENOSPC, &mapping->flags))
  254. ret = -ENOSPC;
  255. if (test_and_clear_bit(AS_EIO, &mapping->flags))
  256. ret = -EIO;
  257. return ret;
  258. }
  259. /**
  260. * sync_page_range - write and wait on all pages in the passed range
  261. * @inode: target inode
  262. * @mapping: target address_space
  263. * @pos: beginning offset in pages to write
  264. * @count: number of bytes to write
  265. *
  266. * Write and wait upon all the pages in the passed range. This is a "data
  267. * integrity" operation. It waits upon in-flight writeout before starting and
  268. * waiting upon new writeout. If there was an IO error, return it.
  269. *
  270. * We need to re-take i_mutex during the generic_osync_inode list walk because
  271. * it is otherwise livelockable.
  272. */
  273. int sync_page_range(struct inode *inode, struct address_space *mapping,
  274. loff_t pos, loff_t count)
  275. {
  276. pgoff_t start = pos >> PAGE_CACHE_SHIFT;
  277. pgoff_t end = (pos + count - 1) >> PAGE_CACHE_SHIFT;
  278. int ret;
  279. if (!mapping_cap_writeback_dirty(mapping) || !count)
  280. return 0;
  281. ret = filemap_fdatawrite_range(mapping, pos, pos + count - 1);
  282. if (ret == 0) {
  283. mutex_lock(&inode->i_mutex);
  284. ret = generic_osync_inode(inode, mapping, OSYNC_METADATA);
  285. mutex_unlock(&inode->i_mutex);
  286. }
  287. if (ret == 0)
  288. ret = wait_on_page_writeback_range(mapping, start, end);
  289. return ret;
  290. }
  291. EXPORT_SYMBOL(sync_page_range);
  292. /**
  293. * sync_page_range_nolock
  294. * @inode: target inode
  295. * @mapping: target address_space
  296. * @pos: beginning offset in pages to write
  297. * @count: number of bytes to write
  298. *
  299. * Note: Holding i_mutex across sync_page_range_nolock is not a good idea
  300. * as it forces O_SYNC writers to different parts of the same file
  301. * to be serialised right until io completion.
  302. */
  303. int sync_page_range_nolock(struct inode *inode, struct address_space *mapping,
  304. loff_t pos, loff_t count)
  305. {
  306. pgoff_t start = pos >> PAGE_CACHE_SHIFT;
  307. pgoff_t end = (pos + count - 1) >> PAGE_CACHE_SHIFT;
  308. int ret;
  309. if (!mapping_cap_writeback_dirty(mapping) || !count)
  310. return 0;
  311. ret = filemap_fdatawrite_range(mapping, pos, pos + count - 1);
  312. if (ret == 0)
  313. ret = generic_osync_inode(inode, mapping, OSYNC_METADATA);
  314. if (ret == 0)
  315. ret = wait_on_page_writeback_range(mapping, start, end);
  316. return ret;
  317. }
  318. EXPORT_SYMBOL(sync_page_range_nolock);
  319. /**
  320. * filemap_fdatawait - wait for all under-writeback pages to complete
  321. * @mapping: address space structure to wait for
  322. *
  323. * Walk the list of under-writeback pages of the given address space
  324. * and wait for all of them.
  325. */
  326. int filemap_fdatawait(struct address_space *mapping)
  327. {
  328. loff_t i_size = i_size_read(mapping->host);
  329. if (i_size == 0)
  330. return 0;
  331. return wait_on_page_writeback_range(mapping, 0,
  332. (i_size - 1) >> PAGE_CACHE_SHIFT);
  333. }
  334. EXPORT_SYMBOL(filemap_fdatawait);
  335. int filemap_write_and_wait(struct address_space *mapping)
  336. {
  337. int err = 0;
  338. if (mapping->nrpages) {
  339. err = filemap_fdatawrite(mapping);
  340. /*
  341. * Even if the above returned error, the pages may be
  342. * written partially (e.g. -ENOSPC), so we wait for it.
  343. * But the -EIO is special case, it may indicate the worst
  344. * thing (e.g. bug) happened, so we avoid waiting for it.
  345. */
  346. if (err != -EIO) {
  347. int err2 = filemap_fdatawait(mapping);
  348. if (!err)
  349. err = err2;
  350. }
  351. }
  352. return err;
  353. }
  354. EXPORT_SYMBOL(filemap_write_and_wait);
  355. /**
  356. * filemap_write_and_wait_range - write out & wait on a file range
  357. * @mapping: the address_space for the pages
  358. * @lstart: offset in bytes where the range starts
  359. * @lend: offset in bytes where the range ends (inclusive)
  360. *
  361. * Write out and wait upon file offsets lstart->lend, inclusive.
  362. *
  363. * Note that `lend' is inclusive (describes the last byte to be written) so
  364. * that this function can be used to write to the very end-of-file (end = -1).
  365. */
  366. int filemap_write_and_wait_range(struct address_space *mapping,
  367. loff_t lstart, loff_t lend)
  368. {
  369. int err = 0;
  370. if (mapping->nrpages) {
  371. err = __filemap_fdatawrite_range(mapping, lstart, lend,
  372. WB_SYNC_ALL);
  373. /* See comment of filemap_write_and_wait() */
  374. if (err != -EIO) {
  375. int err2 = wait_on_page_writeback_range(mapping,
  376. lstart >> PAGE_CACHE_SHIFT,
  377. lend >> PAGE_CACHE_SHIFT);
  378. if (!err)
  379. err = err2;
  380. }
  381. }
  382. return err;
  383. }
  384. /**
  385. * add_to_page_cache - add newly allocated pagecache pages
  386. * @page: page to add
  387. * @mapping: the page's address_space
  388. * @offset: page index
  389. * @gfp_mask: page allocation mode
  390. *
  391. * This function is used to add newly allocated pagecache pages;
  392. * the page is new, so we can just run SetPageLocked() against it.
  393. * The other page state flags were set by rmqueue().
  394. *
  395. * This function does not add the page to the LRU. The caller must do that.
  396. */
  397. int add_to_page_cache(struct page *page, struct address_space *mapping,
  398. pgoff_t offset, gfp_t gfp_mask)
  399. {
  400. int error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM);
  401. if (error == 0) {
  402. write_lock_irq(&mapping->tree_lock);
  403. error = radix_tree_insert(&mapping->page_tree, offset, page);
  404. if (!error) {
  405. page_cache_get(page);
  406. SetPageLocked(page);
  407. page->mapping = mapping;
  408. page->index = offset;
  409. mapping->nrpages++;
  410. pagecache_acct(1);
  411. }
  412. write_unlock_irq(&mapping->tree_lock);
  413. radix_tree_preload_end();
  414. }
  415. return error;
  416. }
  417. EXPORT_SYMBOL(add_to_page_cache);
  418. int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
  419. pgoff_t offset, gfp_t gfp_mask)
  420. {
  421. int ret = add_to_page_cache(page, mapping, offset, gfp_mask);
  422. if (ret == 0)
  423. lru_cache_add(page);
  424. return ret;
  425. }
  426. #ifdef CONFIG_NUMA
  427. struct page *page_cache_alloc(struct address_space *x)
  428. {
  429. if (cpuset_do_page_mem_spread()) {
  430. int n = cpuset_mem_spread_node();
  431. return alloc_pages_node(n, mapping_gfp_mask(x), 0);
  432. }
  433. return alloc_pages(mapping_gfp_mask(x), 0);
  434. }
  435. EXPORT_SYMBOL(page_cache_alloc);
  436. struct page *page_cache_alloc_cold(struct address_space *x)
  437. {
  438. if (cpuset_do_page_mem_spread()) {
  439. int n = cpuset_mem_spread_node();
  440. return alloc_pages_node(n, mapping_gfp_mask(x)|__GFP_COLD, 0);
  441. }
  442. return alloc_pages(mapping_gfp_mask(x)|__GFP_COLD, 0);
  443. }
  444. EXPORT_SYMBOL(page_cache_alloc_cold);
  445. #endif
  446. /*
  447. * In order to wait for pages to become available there must be
  448. * waitqueues associated with pages. By using a hash table of
  449. * waitqueues where the bucket discipline is to maintain all
  450. * waiters on the same queue and wake all when any of the pages
  451. * become available, and for the woken contexts to check to be
  452. * sure the appropriate page became available, this saves space
  453. * at a cost of "thundering herd" phenomena during rare hash
  454. * collisions.
  455. */
  456. static wait_queue_head_t *page_waitqueue(struct page *page)
  457. {
  458. const struct zone *zone = page_zone(page);
  459. return &zone->wait_table[hash_ptr(page, zone->wait_table_bits)];
  460. }
  461. static inline void wake_up_page(struct page *page, int bit)
  462. {
  463. __wake_up_bit(page_waitqueue(page), &page->flags, bit);
  464. }
  465. void fastcall wait_on_page_bit(struct page *page, int bit_nr)
  466. {
  467. DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);
  468. if (test_bit(bit_nr, &page->flags))
  469. __wait_on_bit(page_waitqueue(page), &wait, sync_page,
  470. TASK_UNINTERRUPTIBLE);
  471. }
  472. EXPORT_SYMBOL(wait_on_page_bit);
  473. /**
  474. * unlock_page - unlock a locked page
  475. * @page: the page
  476. *
  477. * Unlocks the page and wakes up sleepers in ___wait_on_page_locked().
  478. * Also wakes sleepers in wait_on_page_writeback() because the wakeup
  479. * mechananism between PageLocked pages and PageWriteback pages is shared.
  480. * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
  481. *
  482. * The first mb is necessary to safely close the critical section opened by the
  483. * TestSetPageLocked(), the second mb is necessary to enforce ordering between
  484. * the clear_bit and the read of the waitqueue (to avoid SMP races with a
  485. * parallel wait_on_page_locked()).
  486. */
  487. void fastcall unlock_page(struct page *page)
  488. {
  489. smp_mb__before_clear_bit();
  490. if (!TestClearPageLocked(page))
  491. BUG();
  492. smp_mb__after_clear_bit();
  493. wake_up_page(page, PG_locked);
  494. }
  495. EXPORT_SYMBOL(unlock_page);
  496. /**
  497. * end_page_writeback - end writeback against a page
  498. * @page: the page
  499. */
  500. void end_page_writeback(struct page *page)
  501. {
  502. if (!TestClearPageReclaim(page) || rotate_reclaimable_page(page)) {
  503. if (!test_clear_page_writeback(page))
  504. BUG();
  505. }
  506. smp_mb__after_clear_bit();
  507. wake_up_page(page, PG_writeback);
  508. }
  509. EXPORT_SYMBOL(end_page_writeback);
  510. /**
  511. * __lock_page - get a lock on the page, assuming we need to sleep to get it
  512. * @page: the page to lock
  513. *
  514. * Ugly. Running sync_page() in state TASK_UNINTERRUPTIBLE is scary. If some
  515. * random driver's requestfn sets TASK_RUNNING, we could busywait. However
  516. * chances are that on the second loop, the block layer's plug list is empty,
  517. * so sync_page() will then return in state TASK_UNINTERRUPTIBLE.
  518. */
  519. void fastcall __lock_page(struct page *page)
  520. {
  521. DEFINE_WAIT_BIT(wait, &page->flags, PG_locked);
  522. __wait_on_bit_lock(page_waitqueue(page), &wait, sync_page,
  523. TASK_UNINTERRUPTIBLE);
  524. }
  525. EXPORT_SYMBOL(__lock_page);
  526. /**
  527. * find_get_page - find and get a page reference
  528. * @mapping: the address_space to search
  529. * @offset: the page index
  530. *
  531. * A rather lightweight function, finding and getting a reference to a
  532. * hashed page atomically.
  533. */
  534. struct page * find_get_page(struct address_space *mapping, unsigned long offset)
  535. {
  536. struct page *page;
  537. read_lock_irq(&mapping->tree_lock);
  538. page = radix_tree_lookup(&mapping->page_tree, offset);
  539. if (page)
  540. page_cache_get(page);
  541. read_unlock_irq(&mapping->tree_lock);
  542. return page;
  543. }
  544. EXPORT_SYMBOL(find_get_page);
  545. /**
  546. * find_trylock_page - find and lock a page
  547. * @mapping: the address_space to search
  548. * @offset: the page index
  549. *
  550. * Same as find_get_page(), but trylock it instead of incrementing the count.
  551. */
  552. struct page *find_trylock_page(struct address_space *mapping, unsigned long offset)
  553. {
  554. struct page *page;
  555. read_lock_irq(&mapping->tree_lock);
  556. page = radix_tree_lookup(&mapping->page_tree, offset);
  557. if (page && TestSetPageLocked(page))
  558. page = NULL;
  559. read_unlock_irq(&mapping->tree_lock);
  560. return page;
  561. }
  562. EXPORT_SYMBOL(find_trylock_page);
  563. /**
  564. * find_lock_page - locate, pin and lock a pagecache page
  565. * @mapping: the address_space to search
  566. * @offset: the page index
  567. *
  568. * Locates the desired pagecache page, locks it, increments its reference
  569. * count and returns its address.
  570. *
  571. * Returns zero if the page was not present. find_lock_page() may sleep.
  572. */
  573. struct page *find_lock_page(struct address_space *mapping,
  574. unsigned long offset)
  575. {
  576. struct page *page;
  577. read_lock_irq(&mapping->tree_lock);
  578. repeat:
  579. page = radix_tree_lookup(&mapping->page_tree, offset);
  580. if (page) {
  581. page_cache_get(page);
  582. if (TestSetPageLocked(page)) {
  583. read_unlock_irq(&mapping->tree_lock);
  584. __lock_page(page);
  585. read_lock_irq(&mapping->tree_lock);
  586. /* Has the page been truncated while we slept? */
  587. if (unlikely(page->mapping != mapping ||
  588. page->index != offset)) {
  589. unlock_page(page);
  590. page_cache_release(page);
  591. goto repeat;
  592. }
  593. }
  594. }
  595. read_unlock_irq(&mapping->tree_lock);
  596. return page;
  597. }
  598. EXPORT_SYMBOL(find_lock_page);
  599. /**
  600. * find_or_create_page - locate or add a pagecache page
  601. * @mapping: the page's address_space
  602. * @index: the page's index into the mapping
  603. * @gfp_mask: page allocation mode
  604. *
  605. * Locates a page in the pagecache. If the page is not present, a new page
  606. * is allocated using @gfp_mask and is added to the pagecache and to the VM's
  607. * LRU list. The returned page is locked and has its reference count
  608. * incremented.
  609. *
  610. * find_or_create_page() may sleep, even if @gfp_flags specifies an atomic
  611. * allocation!
  612. *
  613. * find_or_create_page() returns the desired page's address, or zero on
  614. * memory exhaustion.
  615. */
  616. struct page *find_or_create_page(struct address_space *mapping,
  617. unsigned long index, gfp_t gfp_mask)
  618. {
  619. struct page *page, *cached_page = NULL;
  620. int err;
  621. repeat:
  622. page = find_lock_page(mapping, index);
  623. if (!page) {
  624. if (!cached_page) {
  625. cached_page = alloc_page(gfp_mask);
  626. if (!cached_page)
  627. return NULL;
  628. }
  629. err = add_to_page_cache_lru(cached_page, mapping,
  630. index, gfp_mask);
  631. if (!err) {
  632. page = cached_page;
  633. cached_page = NULL;
  634. } else if (err == -EEXIST)
  635. goto repeat;
  636. }
  637. if (cached_page)
  638. page_cache_release(cached_page);
  639. return page;
  640. }
  641. EXPORT_SYMBOL(find_or_create_page);
  642. /**
  643. * find_get_pages - gang pagecache lookup
  644. * @mapping: The address_space to search
  645. * @start: The starting page index
  646. * @nr_pages: The maximum number of pages
  647. * @pages: Where the resulting pages are placed
  648. *
  649. * find_get_pages() will search for and return a group of up to
  650. * @nr_pages pages in the mapping. The pages are placed at @pages.
  651. * find_get_pages() takes a reference against the returned pages.
  652. *
  653. * The search returns a group of mapping-contiguous pages with ascending
  654. * indexes. There may be holes in the indices due to not-present pages.
  655. *
  656. * find_get_pages() returns the number of pages which were found.
  657. */
  658. unsigned find_get_pages(struct address_space *mapping, pgoff_t start,
  659. unsigned int nr_pages, struct page **pages)
  660. {
  661. unsigned int i;
  662. unsigned int ret;
  663. read_lock_irq(&mapping->tree_lock);
  664. ret = radix_tree_gang_lookup(&mapping->page_tree,
  665. (void **)pages, start, nr_pages);
  666. for (i = 0; i < ret; i++)
  667. page_cache_get(pages[i]);
  668. read_unlock_irq(&mapping->tree_lock);
  669. return ret;
  670. }
  671. /**
  672. * find_get_pages_contig - gang contiguous pagecache lookup
  673. * @mapping: The address_space to search
  674. * @index: The starting page index
  675. * @nr_pages: The maximum number of pages
  676. * @pages: Where the resulting pages are placed
  677. *
  678. * find_get_pages_contig() works exactly like find_get_pages(), except
  679. * that the returned number of pages are guaranteed to be contiguous.
  680. *
  681. * find_get_pages_contig() returns the number of pages which were found.
  682. */
  683. unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
  684. unsigned int nr_pages, struct page **pages)
  685. {
  686. unsigned int i;
  687. unsigned int ret;
  688. read_lock_irq(&mapping->tree_lock);
  689. ret = radix_tree_gang_lookup(&mapping->page_tree,
  690. (void **)pages, index, nr_pages);
  691. for (i = 0; i < ret; i++) {
  692. if (pages[i]->mapping == NULL || pages[i]->index != index)
  693. break;
  694. page_cache_get(pages[i]);
  695. index++;
  696. }
  697. read_unlock_irq(&mapping->tree_lock);
  698. return i;
  699. }
  700. /**
  701. * find_get_pages_tag - find and return pages that match @tag
  702. * @mapping: the address_space to search
  703. * @index: the starting page index
  704. * @tag: the tag index
  705. * @nr_pages: the maximum number of pages
  706. * @pages: where the resulting pages are placed
  707. *
  708. * Like find_get_pages, except we only return pages which are tagged with
  709. * @tag. We update @index to index the next page for the traversal.
  710. */
  711. unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index,
  712. int tag, unsigned int nr_pages, struct page **pages)
  713. {
  714. unsigned int i;
  715. unsigned int ret;
  716. read_lock_irq(&mapping->tree_lock);
  717. ret = radix_tree_gang_lookup_tag(&mapping->page_tree,
  718. (void **)pages, *index, nr_pages, tag);
  719. for (i = 0; i < ret; i++)
  720. page_cache_get(pages[i]);
  721. if (ret)
  722. *index = pages[ret - 1]->index + 1;
  723. read_unlock_irq(&mapping->tree_lock);
  724. return ret;
  725. }
  726. /**
  727. * grab_cache_page_nowait - returns locked page at given index in given cache
  728. * @mapping: target address_space
  729. * @index: the page index
  730. *
  731. * Same as grab_cache_page, but do not wait if the page is unavailable.
  732. * This is intended for speculative data generators, where the data can
  733. * be regenerated if the page couldn't be grabbed. This routine should
  734. * be safe to call while holding the lock for another page.
  735. *
  736. * Clear __GFP_FS when allocating the page to avoid recursion into the fs
  737. * and deadlock against the caller's locked page.
  738. */
  739. struct page *
  740. grab_cache_page_nowait(struct address_space *mapping, unsigned long index)
  741. {
  742. struct page *page = find_get_page(mapping, index);
  743. gfp_t gfp_mask;
  744. if (page) {
  745. if (!TestSetPageLocked(page))
  746. return page;
  747. page_cache_release(page);
  748. return NULL;
  749. }
  750. gfp_mask = mapping_gfp_mask(mapping) & ~__GFP_FS;
  751. page = alloc_pages(gfp_mask, 0);
  752. if (page && add_to_page_cache_lru(page, mapping, index, gfp_mask)) {
  753. page_cache_release(page);
  754. page = NULL;
  755. }
  756. return page;
  757. }
  758. EXPORT_SYMBOL(grab_cache_page_nowait);
  759. /**
  760. * do_generic_mapping_read - generic file read routine
  761. * @mapping: address_space to be read
  762. * @_ra: file's readahead state
  763. * @filp: the file to read
  764. * @ppos: current file position
  765. * @desc: read_descriptor
  766. * @actor: read method
  767. *
  768. * This is a generic file read routine, and uses the
  769. * mapping->a_ops->readpage() function for the actual low-level stuff.
  770. *
  771. * This is really ugly. But the goto's actually try to clarify some
  772. * of the logic when it comes to error handling etc.
  773. *
  774. * Note the struct file* is only passed for the use of readpage.
  775. * It may be NULL.
  776. */
  777. void do_generic_mapping_read(struct address_space *mapping,
  778. struct file_ra_state *_ra,
  779. struct file *filp,
  780. loff_t *ppos,
  781. read_descriptor_t *desc,
  782. read_actor_t actor)
  783. {
  784. struct inode *inode = mapping->host;
  785. unsigned long index;
  786. unsigned long end_index;
  787. unsigned long offset;
  788. unsigned long last_index;
  789. unsigned long next_index;
  790. unsigned long prev_index;
  791. loff_t isize;
  792. struct page *cached_page;
  793. int error;
  794. struct file_ra_state ra = *_ra;
  795. cached_page = NULL;
  796. index = *ppos >> PAGE_CACHE_SHIFT;
  797. next_index = index;
  798. prev_index = ra.prev_page;
  799. last_index = (*ppos + desc->count + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT;
  800. offset = *ppos & ~PAGE_CACHE_MASK;
  801. isize = i_size_read(inode);
  802. if (!isize)
  803. goto out;
  804. end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
  805. for (;;) {
  806. struct page *page;
  807. unsigned long nr, ret;
  808. /* nr is the maximum number of bytes to copy from this page */
  809. nr = PAGE_CACHE_SIZE;
  810. if (index >= end_index) {
  811. if (index > end_index)
  812. goto out;
  813. nr = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
  814. if (nr <= offset) {
  815. goto out;
  816. }
  817. }
  818. nr = nr - offset;
  819. cond_resched();
  820. if (index == next_index)
  821. next_index = page_cache_readahead(mapping, &ra, filp,
  822. index, last_index - index);
  823. find_page:
  824. page = find_get_page(mapping, index);
  825. if (unlikely(page == NULL)) {
  826. handle_ra_miss(mapping, &ra, index);
  827. goto no_cached_page;
  828. }
  829. if (!PageUptodate(page))
  830. goto page_not_up_to_date;
  831. page_ok:
  832. /* If users can be writing to this page using arbitrary
  833. * virtual addresses, take care about potential aliasing
  834. * before reading the page on the kernel side.
  835. */
  836. if (mapping_writably_mapped(mapping))
  837. flush_dcache_page(page);
  838. /*
  839. * When (part of) the same page is read multiple times
  840. * in succession, only mark it as accessed the first time.
  841. */
  842. if (prev_index != index)
  843. mark_page_accessed(page);
  844. prev_index = index;
  845. /*
  846. * Ok, we have the page, and it's up-to-date, so
  847. * now we can copy it to user space...
  848. *
  849. * The actor routine returns how many bytes were actually used..
  850. * NOTE! This may not be the same as how much of a user buffer
  851. * we filled up (we may be padding etc), so we can only update
  852. * "pos" here (the actor routine has to update the user buffer
  853. * pointers and the remaining count).
  854. */
  855. ret = actor(desc, page, offset, nr);
  856. offset += ret;
  857. index += offset >> PAGE_CACHE_SHIFT;
  858. offset &= ~PAGE_CACHE_MASK;
  859. page_cache_release(page);
  860. if (ret == nr && desc->count)
  861. continue;
  862. goto out;
  863. page_not_up_to_date:
  864. /* Get exclusive access to the page ... */
  865. lock_page(page);
  866. /* Did it get unhashed before we got the lock? */
  867. if (!page->mapping) {
  868. unlock_page(page);
  869. page_cache_release(page);
  870. continue;
  871. }
  872. /* Did somebody else fill it already? */
  873. if (PageUptodate(page)) {
  874. unlock_page(page);
  875. goto page_ok;
  876. }
  877. readpage:
  878. /* Start the actual read. The read will unlock the page. */
  879. error = mapping->a_ops->readpage(filp, page);
  880. if (unlikely(error)) {
  881. if (error == AOP_TRUNCATED_PAGE) {
  882. page_cache_release(page);
  883. goto find_page;
  884. }
  885. goto readpage_error;
  886. }
  887. if (!PageUptodate(page)) {
  888. lock_page(page);
  889. if (!PageUptodate(page)) {
  890. if (page->mapping == NULL) {
  891. /*
  892. * invalidate_inode_pages got it
  893. */
  894. unlock_page(page);
  895. page_cache_release(page);
  896. goto find_page;
  897. }
  898. unlock_page(page);
  899. error = -EIO;
  900. goto readpage_error;
  901. }
  902. unlock_page(page);
  903. }
  904. /*
  905. * i_size must be checked after we have done ->readpage.
  906. *
  907. * Checking i_size after the readpage allows us to calculate
  908. * the correct value for "nr", which means the zero-filled
  909. * part of the page is not copied back to userspace (unless
  910. * another truncate extends the file - this is desired though).
  911. */
  912. isize = i_size_read(inode);
  913. end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
  914. if (unlikely(!isize || index > end_index)) {
  915. page_cache_release(page);
  916. goto out;
  917. }
  918. /* nr is the maximum number of bytes to copy from this page */
  919. nr = PAGE_CACHE_SIZE;
  920. if (index == end_index) {
  921. nr = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
  922. if (nr <= offset) {
  923. page_cache_release(page);
  924. goto out;
  925. }
  926. }
  927. nr = nr - offset;
  928. goto page_ok;
  929. readpage_error:
  930. /* UHHUH! A synchronous read error occurred. Report it */
  931. desc->error = error;
  932. page_cache_release(page);
  933. goto out;
  934. no_cached_page:
  935. /*
  936. * Ok, it wasn't cached, so we need to create a new
  937. * page..
  938. */
  939. if (!cached_page) {
  940. cached_page = page_cache_alloc_cold(mapping);
  941. if (!cached_page) {
  942. desc->error = -ENOMEM;
  943. goto out;
  944. }
  945. }
  946. error = add_to_page_cache_lru(cached_page, mapping,
  947. index, GFP_KERNEL);
  948. if (error) {
  949. if (error == -EEXIST)
  950. goto find_page;
  951. desc->error = error;
  952. goto out;
  953. }
  954. page = cached_page;
  955. cached_page = NULL;
  956. goto readpage;
  957. }
  958. out:
  959. *_ra = ra;
  960. *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
  961. if (cached_page)
  962. page_cache_release(cached_page);
  963. if (filp)
  964. file_accessed(filp);
  965. }
  966. EXPORT_SYMBOL(do_generic_mapping_read);
  967. int file_read_actor(read_descriptor_t *desc, struct page *page,
  968. unsigned long offset, unsigned long size)
  969. {
  970. char *kaddr;
  971. unsigned long left, count = desc->count;
  972. if (size > count)
  973. size = count;
  974. /*
  975. * Faults on the destination of a read are common, so do it before
  976. * taking the kmap.
  977. */
  978. if (!fault_in_pages_writeable(desc->arg.buf, size)) {
  979. kaddr = kmap_atomic(page, KM_USER0);
  980. left = __copy_to_user_inatomic(desc->arg.buf,
  981. kaddr + offset, size);
  982. kunmap_atomic(kaddr, KM_USER0);
  983. if (left == 0)
  984. goto success;
  985. }
  986. /* Do it the slow way */
  987. kaddr = kmap(page);
  988. left = __copy_to_user(desc->arg.buf, kaddr + offset, size);
  989. kunmap(page);
  990. if (left) {
  991. size -= left;
  992. desc->error = -EFAULT;
  993. }
  994. success:
  995. desc->count = count - size;
  996. desc->written += size;
  997. desc->arg.buf += size;
  998. return size;
  999. }
  1000. /**
  1001. * __generic_file_aio_read - generic filesystem read routine
  1002. * @iocb: kernel I/O control block
  1003. * @iov: io vector request
  1004. * @nr_segs: number of segments in the iovec
  1005. * @ppos: current file position
  1006. *
  1007. * This is the "read()" routine for all filesystems
  1008. * that can use the page cache directly.
  1009. */
  1010. ssize_t
  1011. __generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
  1012. unsigned long nr_segs, loff_t *ppos)
  1013. {
  1014. struct file *filp = iocb->ki_filp;
  1015. ssize_t retval;
  1016. unsigned long seg;
  1017. size_t count;
  1018. count = 0;
  1019. for (seg = 0; seg < nr_segs; seg++) {
  1020. const struct iovec *iv = &iov[seg];
  1021. /*
  1022. * If any segment has a negative length, or the cumulative
  1023. * length ever wraps negative then return -EINVAL.
  1024. */
  1025. count += iv->iov_len;
  1026. if (unlikely((ssize_t)(count|iv->iov_len) < 0))
  1027. return -EINVAL;
  1028. if (access_ok(VERIFY_WRITE, iv->iov_base, iv->iov_len))
  1029. continue;
  1030. if (seg == 0)
  1031. return -EFAULT;
  1032. nr_segs = seg;
  1033. count -= iv->iov_len; /* This segment is no good */
  1034. break;
  1035. }
  1036. /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
  1037. if (filp->f_flags & O_DIRECT) {
  1038. loff_t pos = *ppos, size;
  1039. struct address_space *mapping;
  1040. struct inode *inode;
  1041. mapping = filp->f_mapping;
  1042. inode = mapping->host;
  1043. retval = 0;
  1044. if (!count)
  1045. goto out; /* skip atime */
  1046. size = i_size_read(inode);
  1047. if (pos < size) {
  1048. retval = generic_file_direct_IO(READ, iocb,
  1049. iov, pos, nr_segs);
  1050. if (retval > 0 && !is_sync_kiocb(iocb))
  1051. retval = -EIOCBQUEUED;
  1052. if (retval > 0)
  1053. *ppos = pos + retval;
  1054. }
  1055. file_accessed(filp);
  1056. goto out;
  1057. }
  1058. retval = 0;
  1059. if (count) {
  1060. for (seg = 0; seg < nr_segs; seg++) {
  1061. read_descriptor_t desc;
  1062. desc.written = 0;
  1063. desc.arg.buf = iov[seg].iov_base;
  1064. desc.count = iov[seg].iov_len;
  1065. if (desc.count == 0)
  1066. continue;
  1067. desc.error = 0;
  1068. do_generic_file_read(filp,ppos,&desc,file_read_actor);
  1069. retval += desc.written;
  1070. if (desc.error) {
  1071. retval = retval ?: desc.error;
  1072. break;
  1073. }
  1074. }
  1075. }
  1076. out:
  1077. return retval;
  1078. }
  1079. EXPORT_SYMBOL(__generic_file_aio_read);
  1080. ssize_t
  1081. generic_file_aio_read(struct kiocb *iocb, char __user *buf, size_t count, loff_t pos)
  1082. {
  1083. struct iovec local_iov = { .iov_base = buf, .iov_len = count };
  1084. BUG_ON(iocb->ki_pos != pos);
  1085. return __generic_file_aio_read(iocb, &local_iov, 1, &iocb->ki_pos);
  1086. }
  1087. EXPORT_SYMBOL(generic_file_aio_read);
  1088. ssize_t
  1089. generic_file_read(struct file *filp, char __user *buf, size_t count, loff_t *ppos)
  1090. {
  1091. struct iovec local_iov = { .iov_base = buf, .iov_len = count };
  1092. struct kiocb kiocb;
  1093. ssize_t ret;
  1094. init_sync_kiocb(&kiocb, filp);
  1095. ret = __generic_file_aio_read(&kiocb, &local_iov, 1, ppos);
  1096. if (-EIOCBQUEUED == ret)
  1097. ret = wait_on_sync_kiocb(&kiocb);
  1098. return ret;
  1099. }
  1100. EXPORT_SYMBOL(generic_file_read);
  1101. int file_send_actor(read_descriptor_t * desc, struct page *page, unsigned long offset, unsigned long size)
  1102. {
  1103. ssize_t written;
  1104. unsigned long count = desc->count;
  1105. struct file *file = desc->arg.data;
  1106. if (size > count)
  1107. size = count;
  1108. written = file->f_op->sendpage(file, page, offset,
  1109. size, &file->f_pos, size<count);
  1110. if (written < 0) {
  1111. desc->error = written;
  1112. written = 0;
  1113. }
  1114. desc->count = count - written;
  1115. desc->written += written;
  1116. return written;
  1117. }
  1118. ssize_t generic_file_sendfile(struct file *in_file, loff_t *ppos,
  1119. size_t count, read_actor_t actor, void *target)
  1120. {
  1121. read_descriptor_t desc;
  1122. if (!count)
  1123. return 0;
  1124. desc.written = 0;
  1125. desc.count = count;
  1126. desc.arg.data = target;
  1127. desc.error = 0;
  1128. do_generic_file_read(in_file, ppos, &desc, actor);
  1129. if (desc.written)
  1130. return desc.written;
  1131. return desc.error;
  1132. }
  1133. EXPORT_SYMBOL(generic_file_sendfile);
  1134. static ssize_t
  1135. do_readahead(struct address_space *mapping, struct file *filp,
  1136. unsigned long index, unsigned long nr)
  1137. {
  1138. if (!mapping || !mapping->a_ops || !mapping->a_ops->readpage)
  1139. return -EINVAL;
  1140. force_page_cache_readahead(mapping, filp, index,
  1141. max_sane_readahead(nr));
  1142. return 0;
  1143. }
  1144. asmlinkage ssize_t sys_readahead(int fd, loff_t offset, size_t count)
  1145. {
  1146. ssize_t ret;
  1147. struct file *file;
  1148. ret = -EBADF;
  1149. file = fget(fd);
  1150. if (file) {
  1151. if (file->f_mode & FMODE_READ) {
  1152. struct address_space *mapping = file->f_mapping;
  1153. unsigned long start = offset >> PAGE_CACHE_SHIFT;
  1154. unsigned long end = (offset + count - 1) >> PAGE_CACHE_SHIFT;
  1155. unsigned long len = end - start + 1;
  1156. ret = do_readahead(mapping, file, start, len);
  1157. }
  1158. fput(file);
  1159. }
  1160. return ret;
  1161. }
  1162. #ifdef CONFIG_MMU
  1163. static int FASTCALL(page_cache_read(struct file * file, unsigned long offset));
  1164. /**
  1165. * page_cache_read - adds requested page to the page cache if not already there
  1166. * @file: file to read
  1167. * @offset: page index
  1168. *
  1169. * This adds the requested page to the page cache if it isn't already there,
  1170. * and schedules an I/O to read in its contents from disk.
  1171. */
  1172. static int fastcall page_cache_read(struct file * file, unsigned long offset)
  1173. {
  1174. struct address_space *mapping = file->f_mapping;
  1175. struct page *page;
  1176. int ret;
  1177. do {
  1178. page = page_cache_alloc_cold(mapping);
  1179. if (!page)
  1180. return -ENOMEM;
  1181. ret = add_to_page_cache_lru(page, mapping, offset, GFP_KERNEL);
  1182. if (ret == 0)
  1183. ret = mapping->a_ops->readpage(file, page);
  1184. else if (ret == -EEXIST)
  1185. ret = 0; /* losing race to add is OK */
  1186. page_cache_release(page);
  1187. } while (ret == AOP_TRUNCATED_PAGE);
  1188. return ret;
  1189. }
  1190. #define MMAP_LOTSAMISS (100)
  1191. /**
  1192. * filemap_nopage - read in file data for page fault handling
  1193. * @area: the applicable vm_area
  1194. * @address: target address to read in
  1195. * @type: returned with VM_FAULT_{MINOR,MAJOR} if not %NULL
  1196. *
  1197. * filemap_nopage() is invoked via the vma operations vector for a
  1198. * mapped memory region to read in file data during a page fault.
  1199. *
  1200. * The goto's are kind of ugly, but this streamlines the normal case of having
  1201. * it in the page cache, and handles the special cases reasonably without
  1202. * having a lot of duplicated code.
  1203. */
  1204. struct page *filemap_nopage(struct vm_area_struct *area,
  1205. unsigned long address, int *type)
  1206. {
  1207. int error;
  1208. struct file *file = area->vm_file;
  1209. struct address_space *mapping = file->f_mapping;
  1210. struct file_ra_state *ra = &file->f_ra;
  1211. struct inode *inode = mapping->host;
  1212. struct page *page;
  1213. unsigned long size, pgoff;
  1214. int did_readaround = 0, majmin = VM_FAULT_MINOR;
  1215. pgoff = ((address-area->vm_start) >> PAGE_CACHE_SHIFT) + area->vm_pgoff;
  1216. retry_all:
  1217. size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
  1218. if (pgoff >= size)
  1219. goto outside_data_content;
  1220. /* If we don't want any read-ahead, don't bother */
  1221. if (VM_RandomReadHint(area))
  1222. goto no_cached_page;
  1223. /*
  1224. * The readahead code wants to be told about each and every page
  1225. * so it can build and shrink its windows appropriately
  1226. *
  1227. * For sequential accesses, we use the generic readahead logic.
  1228. */
  1229. if (VM_SequentialReadHint(area))
  1230. page_cache_readahead(mapping, ra, file, pgoff, 1);
  1231. /*
  1232. * Do we have something in the page cache already?
  1233. */
  1234. retry_find:
  1235. page = find_get_page(mapping, pgoff);
  1236. if (!page) {
  1237. unsigned long ra_pages;
  1238. if (VM_SequentialReadHint(area)) {
  1239. handle_ra_miss(mapping, ra, pgoff);
  1240. goto no_cached_page;
  1241. }
  1242. ra->mmap_miss++;
  1243. /*
  1244. * Do we miss much more than hit in this file? If so,
  1245. * stop bothering with read-ahead. It will only hurt.
  1246. */
  1247. if (ra->mmap_miss > ra->mmap_hit + MMAP_LOTSAMISS)
  1248. goto no_cached_page;
  1249. /*
  1250. * To keep the pgmajfault counter straight, we need to
  1251. * check did_readaround, as this is an inner loop.
  1252. */
  1253. if (!did_readaround) {
  1254. majmin = VM_FAULT_MAJOR;
  1255. inc_page_state(pgmajfault);
  1256. }
  1257. did_readaround = 1;
  1258. ra_pages = max_sane_readahead(file->f_ra.ra_pages);
  1259. if (ra_pages) {
  1260. pgoff_t start = 0;
  1261. if (pgoff > ra_pages / 2)
  1262. start = pgoff - ra_pages / 2;
  1263. do_page_cache_readahead(mapping, file, start, ra_pages);
  1264. }
  1265. page = find_get_page(mapping, pgoff);
  1266. if (!page)
  1267. goto no_cached_page;
  1268. }
  1269. if (!did_readaround)
  1270. ra->mmap_hit++;
  1271. /*
  1272. * Ok, found a page in the page cache, now we need to check
  1273. * that it's up-to-date.
  1274. */
  1275. if (!PageUptodate(page))
  1276. goto page_not_uptodate;
  1277. success:
  1278. /*
  1279. * Found the page and have a reference on it.
  1280. */
  1281. mark_page_accessed(page);
  1282. if (type)
  1283. *type = majmin;
  1284. return page;
  1285. outside_data_content:
  1286. /*
  1287. * An external ptracer can access pages that normally aren't
  1288. * accessible..
  1289. */
  1290. if (area->vm_mm == current->mm)
  1291. return NULL;
  1292. /* Fall through to the non-read-ahead case */
  1293. no_cached_page:
  1294. /*
  1295. * We're only likely to ever get here if MADV_RANDOM is in
  1296. * effect.
  1297. */
  1298. error = page_cache_read(file, pgoff);
  1299. grab_swap_token();
  1300. /*
  1301. * The page we want has now been added to the page cache.
  1302. * In the unlikely event that someone removed it in the
  1303. * meantime, we'll just come back here and read it again.
  1304. */
  1305. if (error >= 0)
  1306. goto retry_find;
  1307. /*
  1308. * An error return from page_cache_read can result if the
  1309. * system is low on memory, or a problem occurs while trying
  1310. * to schedule I/O.
  1311. */
  1312. if (error == -ENOMEM)
  1313. return NOPAGE_OOM;
  1314. return NULL;
  1315. page_not_uptodate:
  1316. if (!did_readaround) {
  1317. majmin = VM_FAULT_MAJOR;
  1318. inc_page_state(pgmajfault);
  1319. }
  1320. lock_page(page);
  1321. /* Did it get unhashed while we waited for it? */
  1322. if (!page->mapping) {
  1323. unlock_page(page);
  1324. page_cache_release(page);
  1325. goto retry_all;
  1326. }
  1327. /* Did somebody else get it up-to-date? */
  1328. if (PageUptodate(page)) {
  1329. unlock_page(page);
  1330. goto success;
  1331. }
  1332. error = mapping->a_ops->readpage(file, page);
  1333. if (!error) {
  1334. wait_on_page_locked(page);
  1335. if (PageUptodate(page))
  1336. goto success;
  1337. } else if (error == AOP_TRUNCATED_PAGE) {
  1338. page_cache_release(page);
  1339. goto retry_find;
  1340. }
  1341. /*
  1342. * Umm, take care of errors if the page isn't up-to-date.
  1343. * Try to re-read it _once_. We do this synchronously,
  1344. * because there really aren't any performance issues here
  1345. * and we need to check for errors.
  1346. */
  1347. lock_page(page);
  1348. /* Somebody truncated the page on us? */
  1349. if (!page->mapping) {
  1350. unlock_page(page);
  1351. page_cache_release(page);
  1352. goto retry_all;
  1353. }
  1354. /* Somebody else successfully read it in? */
  1355. if (PageUptodate(page)) {
  1356. unlock_page(page);
  1357. goto success;
  1358. }
  1359. ClearPageError(page);
  1360. error = mapping->a_ops->readpage(file, page);
  1361. if (!error) {
  1362. wait_on_page_locked(page);
  1363. if (PageUptodate(page))
  1364. goto success;
  1365. } else if (error == AOP_TRUNCATED_PAGE) {
  1366. page_cache_release(page);
  1367. goto retry_find;
  1368. }
  1369. /*
  1370. * Things didn't work out. Return zero to tell the
  1371. * mm layer so, possibly freeing the page cache page first.
  1372. */
  1373. page_cache_release(page);
  1374. return NULL;
  1375. }
  1376. EXPORT_SYMBOL(filemap_nopage);
  1377. static struct page * filemap_getpage(struct file *file, unsigned long pgoff,
  1378. int nonblock)
  1379. {
  1380. struct address_space *mapping = file->f_mapping;
  1381. struct page *page;
  1382. int error;
  1383. /*
  1384. * Do we have something in the page cache already?
  1385. */
  1386. retry_find:
  1387. page = find_get_page(mapping, pgoff);
  1388. if (!page) {
  1389. if (nonblock)
  1390. return NULL;
  1391. goto no_cached_page;
  1392. }
  1393. /*
  1394. * Ok, found a page in the page cache, now we need to check
  1395. * that it's up-to-date.
  1396. */
  1397. if (!PageUptodate(page)) {
  1398. if (nonblock) {
  1399. page_cache_release(page);
  1400. return NULL;
  1401. }
  1402. goto page_not_uptodate;
  1403. }
  1404. success:
  1405. /*
  1406. * Found the page and have a reference on it.
  1407. */
  1408. mark_page_accessed(page);
  1409. return page;
  1410. no_cached_page:
  1411. error = page_cache_read(file, pgoff);
  1412. /*
  1413. * The page we want has now been added to the page cache.
  1414. * In the unlikely event that someone removed it in the
  1415. * meantime, we'll just come back here and read it again.
  1416. */
  1417. if (error >= 0)
  1418. goto retry_find;
  1419. /*
  1420. * An error return from page_cache_read can result if the
  1421. * system is low on memory, or a problem occurs while trying
  1422. * to schedule I/O.
  1423. */
  1424. return NULL;
  1425. page_not_uptodate:
  1426. lock_page(page);
  1427. /* Did it get unhashed while we waited for it? */
  1428. if (!page->mapping) {
  1429. unlock_page(page);
  1430. goto err;
  1431. }
  1432. /* Did somebody else get it up-to-date? */
  1433. if (PageUptodate(page)) {
  1434. unlock_page(page);
  1435. goto success;
  1436. }
  1437. error = mapping->a_ops->readpage(file, page);
  1438. if (!error) {
  1439. wait_on_page_locked(page);
  1440. if (PageUptodate(page))
  1441. goto success;
  1442. } else if (error == AOP_TRUNCATED_PAGE) {
  1443. page_cache_release(page);
  1444. goto retry_find;
  1445. }
  1446. /*
  1447. * Umm, take care of errors if the page isn't up-to-date.
  1448. * Try to re-read it _once_. We do this synchronously,
  1449. * because there really aren't any performance issues here
  1450. * and we need to check for errors.
  1451. */
  1452. lock_page(page);
  1453. /* Somebody truncated the page on us? */
  1454. if (!page->mapping) {
  1455. unlock_page(page);
  1456. goto err;
  1457. }
  1458. /* Somebody else successfully read it in? */
  1459. if (PageUptodate(page)) {
  1460. unlock_page(page);
  1461. goto success;
  1462. }
  1463. ClearPageError(page);
  1464. error = mapping->a_ops->readpage(file, page);
  1465. if (!error) {
  1466. wait_on_page_locked(page);
  1467. if (PageUptodate(page))
  1468. goto success;
  1469. } else if (error == AOP_TRUNCATED_PAGE) {
  1470. page_cache_release(page);
  1471. goto retry_find;
  1472. }
  1473. /*
  1474. * Things didn't work out. Return zero to tell the
  1475. * mm layer so, possibly freeing the page cache page first.
  1476. */
  1477. err:
  1478. page_cache_release(page);
  1479. return NULL;
  1480. }
  1481. int filemap_populate(struct vm_area_struct *vma, unsigned long addr,
  1482. unsigned long len, pgprot_t prot, unsigned long pgoff,
  1483. int nonblock)
  1484. {
  1485. struct file *file = vma->vm_file;
  1486. struct address_space *mapping = file->f_mapping;
  1487. struct inode *inode = mapping->host;
  1488. unsigned long size;
  1489. struct mm_struct *mm = vma->vm_mm;
  1490. struct page *page;
  1491. int err;
  1492. if (!nonblock)
  1493. force_page_cache_readahead(mapping, vma->vm_file,
  1494. pgoff, len >> PAGE_CACHE_SHIFT);
  1495. repeat:
  1496. size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
  1497. if (pgoff + (len >> PAGE_CACHE_SHIFT) > size)
  1498. return -EINVAL;
  1499. page = filemap_getpage(file, pgoff, nonblock);
  1500. /* XXX: This is wrong, a filesystem I/O error may have happened. Fix that as
  1501. * done in shmem_populate calling shmem_getpage */
  1502. if (!page && !nonblock)
  1503. return -ENOMEM;
  1504. if (page) {
  1505. err = install_page(mm, vma, addr, page, prot);
  1506. if (err) {
  1507. page_cache_release(page);
  1508. return err;
  1509. }
  1510. } else if (vma->vm_flags & VM_NONLINEAR) {
  1511. /* No page was found just because we can't read it in now (being
  1512. * here implies nonblock != 0), but the page may exist, so set
  1513. * the PTE to fault it in later. */
  1514. err = install_file_pte(mm, vma, addr, pgoff, prot);
  1515. if (err)
  1516. return err;
  1517. }
  1518. len -= PAGE_SIZE;
  1519. addr += PAGE_SIZE;
  1520. pgoff++;
  1521. if (len)
  1522. goto repeat;
  1523. return 0;
  1524. }
  1525. EXPORT_SYMBOL(filemap_populate);
  1526. struct vm_operations_struct generic_file_vm_ops = {
  1527. .nopage = filemap_nopage,
  1528. .populate = filemap_populate,
  1529. };
  1530. /* This is used for a general mmap of a disk file */
  1531. int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
  1532. {
  1533. struct address_space *mapping = file->f_mapping;
  1534. if (!mapping->a_ops->readpage)
  1535. return -ENOEXEC;
  1536. file_accessed(file);
  1537. vma->vm_ops = &generic_file_vm_ops;
  1538. return 0;
  1539. }
  1540. /*
  1541. * This is for filesystems which do not implement ->writepage.
  1542. */
  1543. int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma)
  1544. {
  1545. if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
  1546. return -EINVAL;
  1547. return generic_file_mmap(file, vma);
  1548. }
  1549. #else
  1550. int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
  1551. {
  1552. return -ENOSYS;
  1553. }
  1554. int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma)
  1555. {
  1556. return -ENOSYS;
  1557. }
  1558. #endif /* CONFIG_MMU */
  1559. EXPORT_SYMBOL(generic_file_mmap);
  1560. EXPORT_SYMBOL(generic_file_readonly_mmap);
  1561. static inline struct page *__read_cache_page(struct address_space *mapping,
  1562. unsigned long index,
  1563. int (*filler)(void *,struct page*),
  1564. void *data)
  1565. {
  1566. struct page *page, *cached_page = NULL;
  1567. int err;
  1568. repeat:
  1569. page = find_get_page(mapping, index);
  1570. if (!page) {
  1571. if (!cached_page) {
  1572. cached_page = page_cache_alloc_cold(mapping);
  1573. if (!cached_page)
  1574. return ERR_PTR(-ENOMEM);
  1575. }
  1576. err = add_to_page_cache_lru(cached_page, mapping,
  1577. index, GFP_KERNEL);
  1578. if (err == -EEXIST)
  1579. goto repeat;
  1580. if (err < 0) {
  1581. /* Presumably ENOMEM for radix tree node */
  1582. page_cache_release(cached_page);
  1583. return ERR_PTR(err);
  1584. }
  1585. page = cached_page;
  1586. cached_page = NULL;
  1587. err = filler(data, page);
  1588. if (err < 0) {
  1589. page_cache_release(page);
  1590. page = ERR_PTR(err);
  1591. }
  1592. }
  1593. if (cached_page)
  1594. page_cache_release(cached_page);
  1595. return page;
  1596. }
  1597. /**
  1598. * read_cache_page - read into page cache, fill it if needed
  1599. * @mapping: the page's address_space
  1600. * @index: the page index
  1601. * @filler: function to perform the read
  1602. * @data: destination for read data
  1603. *
  1604. * Read into the page cache. If a page already exists,
  1605. * and PageUptodate() is not set, try to fill the page.
  1606. */
  1607. struct page *read_cache_page(struct address_space *mapping,
  1608. unsigned long index,
  1609. int (*filler)(void *,struct page*),
  1610. void *data)
  1611. {
  1612. struct page *page;
  1613. int err;
  1614. retry:
  1615. page = __read_cache_page(mapping, index, filler, data);
  1616. if (IS_ERR(page))
  1617. goto out;
  1618. mark_page_accessed(page);
  1619. if (PageUptodate(page))
  1620. goto out;
  1621. lock_page(page);
  1622. if (!page->mapping) {
  1623. unlock_page(page);
  1624. page_cache_release(page);
  1625. goto retry;
  1626. }
  1627. if (PageUptodate(page)) {
  1628. unlock_page(page);
  1629. goto out;
  1630. }
  1631. err = filler(data, page);
  1632. if (err < 0) {
  1633. page_cache_release(page);
  1634. page = ERR_PTR(err);
  1635. }
  1636. out:
  1637. return page;
  1638. }
  1639. EXPORT_SYMBOL(read_cache_page);
  1640. /*
  1641. * If the page was newly created, increment its refcount and add it to the
  1642. * caller's lru-buffering pagevec. This function is specifically for
  1643. * generic_file_write().
  1644. */
  1645. static inline struct page *
  1646. __grab_cache_page(struct address_space *mapping, unsigned long index,
  1647. struct page **cached_page, struct pagevec *lru_pvec)
  1648. {
  1649. int err;
  1650. struct page *page;
  1651. repeat:
  1652. page = find_lock_page(mapping, index);
  1653. if (!page) {
  1654. if (!*cached_page) {
  1655. *cached_page = page_cache_alloc(mapping);
  1656. if (!*cached_page)
  1657. return NULL;
  1658. }
  1659. err = add_to_page_cache(*cached_page, mapping,
  1660. index, GFP_KERNEL);
  1661. if (err == -EEXIST)
  1662. goto repeat;
  1663. if (err == 0) {
  1664. page = *cached_page;
  1665. page_cache_get(page);
  1666. if (!pagevec_add(lru_pvec, page))
  1667. __pagevec_lru_add(lru_pvec);
  1668. *cached_page = NULL;
  1669. }
  1670. }
  1671. return page;
  1672. }
  1673. /*
  1674. * The logic we want is
  1675. *
  1676. * if suid or (sgid and xgrp)
  1677. * remove privs
  1678. */
  1679. int remove_suid(struct dentry *dentry)
  1680. {
  1681. mode_t mode = dentry->d_inode->i_mode;
  1682. int kill = 0;
  1683. int result = 0;
  1684. /* suid always must be killed */
  1685. if (unlikely(mode & S_ISUID))
  1686. kill = ATTR_KILL_SUID;
  1687. /*
  1688. * sgid without any exec bits is just a mandatory locking mark; leave
  1689. * it alone. If some exec bits are set, it's a real sgid; kill it.
  1690. */
  1691. if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
  1692. kill |= ATTR_KILL_SGID;
  1693. if (unlikely(kill && !capable(CAP_FSETID))) {
  1694. struct iattr newattrs;
  1695. newattrs.ia_valid = ATTR_FORCE | kill;
  1696. result = notify_change(dentry, &newattrs);
  1697. }
  1698. return result;
  1699. }
  1700. EXPORT_SYMBOL(remove_suid);
  1701. size_t
  1702. __filemap_copy_from_user_iovec(char *vaddr,
  1703. const struct iovec *iov, size_t base, size_t bytes)
  1704. {
  1705. size_t copied = 0, left = 0;
  1706. while (bytes) {
  1707. char __user *buf = iov->iov_base + base;
  1708. int copy = min(bytes, iov->iov_len - base);
  1709. base = 0;
  1710. left = __copy_from_user_inatomic_nocache(vaddr, buf, copy);
  1711. copied += copy;
  1712. bytes -= copy;
  1713. vaddr += copy;
  1714. iov++;
  1715. if (unlikely(left)) {
  1716. /* zero the rest of the target like __copy_from_user */
  1717. if (bytes)
  1718. memset(vaddr, 0, bytes);
  1719. break;
  1720. }
  1721. }
  1722. return copied - left;
  1723. }
  1724. /*
  1725. * Performs necessary checks before doing a write
  1726. *
  1727. * Can adjust writing position or amount of bytes to write.
  1728. * Returns appropriate error code that caller should return or
  1729. * zero in case that write should be allowed.
  1730. */
  1731. inline int generic_write_checks(struct file *file, loff_t *pos, size_t *count, int isblk)
  1732. {
  1733. struct inode *inode = file->f_mapping->host;
  1734. unsigned long limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur;
  1735. if (unlikely(*pos < 0))
  1736. return -EINVAL;
  1737. if (!isblk) {
  1738. /* FIXME: this is for backwards compatibility with 2.4 */
  1739. if (file->f_flags & O_APPEND)
  1740. *pos = i_size_read(inode);
  1741. if (limit != RLIM_INFINITY) {
  1742. if (*pos >= limit) {
  1743. send_sig(SIGXFSZ, current, 0);
  1744. return -EFBIG;
  1745. }
  1746. if (*count > limit - (typeof(limit))*pos) {
  1747. *count = limit - (typeof(limit))*pos;
  1748. }
  1749. }
  1750. }
  1751. /*
  1752. * LFS rule
  1753. */
  1754. if (unlikely(*pos + *count > MAX_NON_LFS &&
  1755. !(file->f_flags & O_LARGEFILE))) {
  1756. if (*pos >= MAX_NON_LFS) {
  1757. send_sig(SIGXFSZ, current, 0);
  1758. return -EFBIG;
  1759. }
  1760. if (*count > MAX_NON_LFS - (unsigned long)*pos) {
  1761. *count = MAX_NON_LFS - (unsigned long)*pos;
  1762. }
  1763. }
  1764. /*
  1765. * Are we about to exceed the fs block limit ?
  1766. *
  1767. * If we have written data it becomes a short write. If we have
  1768. * exceeded without writing data we send a signal and return EFBIG.
  1769. * Linus frestrict idea will clean these up nicely..
  1770. */
  1771. if (likely(!isblk)) {
  1772. if (unlikely(*pos >= inode->i_sb->s_maxbytes)) {
  1773. if (*count || *pos > inode->i_sb->s_maxbytes) {
  1774. send_sig(SIGXFSZ, current, 0);
  1775. return -EFBIG;
  1776. }
  1777. /* zero-length writes at ->s_maxbytes are OK */
  1778. }
  1779. if (unlikely(*pos + *count > inode->i_sb->s_maxbytes))
  1780. *count = inode->i_sb->s_maxbytes - *pos;
  1781. } else {
  1782. loff_t isize;
  1783. if (bdev_read_only(I_BDEV(inode)))
  1784. return -EPERM;
  1785. isize = i_size_read(inode);
  1786. if (*pos >= isize) {
  1787. if (*count || *pos > isize)
  1788. return -ENOSPC;
  1789. }
  1790. if (*pos + *count > isize)
  1791. *count = isize - *pos;
  1792. }
  1793. return 0;
  1794. }
  1795. EXPORT_SYMBOL(generic_write_checks);
  1796. ssize_t
  1797. generic_file_direct_write(struct kiocb *iocb, const struct iovec *iov,
  1798. unsigned long *nr_segs, loff_t pos, loff_t *ppos,
  1799. size_t count, size_t ocount)
  1800. {
  1801. struct file *file = iocb->ki_filp;
  1802. struct address_space *mapping = file->f_mapping;
  1803. struct inode *inode = mapping->host;
  1804. ssize_t written;
  1805. if (count != ocount)
  1806. *nr_segs = iov_shorten((struct iovec *)iov, *nr_segs, count);
  1807. written = generic_file_direct_IO(WRITE, iocb, iov, pos, *nr_segs);
  1808. if (written > 0) {
  1809. loff_t end = pos + written;
  1810. if (end > i_size_read(inode) && !S_ISBLK(inode->i_mode)) {
  1811. i_size_write(inode, end);
  1812. mark_inode_dirty(inode);
  1813. }
  1814. *ppos = end;
  1815. }
  1816. /*
  1817. * Sync the fs metadata but not the minor inode changes and
  1818. * of course not the data as we did direct DMA for the IO.
  1819. * i_mutex is held, which protects generic_osync_inode() from
  1820. * livelocking.
  1821. */
  1822. if (written >= 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) {
  1823. int err = generic_osync_inode(inode, mapping, OSYNC_METADATA);
  1824. if (err < 0)
  1825. written = err;
  1826. }
  1827. if (written == count && !is_sync_kiocb(iocb))
  1828. written = -EIOCBQUEUED;
  1829. return written;
  1830. }
  1831. EXPORT_SYMBOL(generic_file_direct_write);
  1832. ssize_t
  1833. generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
  1834. unsigned long nr_segs, loff_t pos, loff_t *ppos,
  1835. size_t count, ssize_t written)
  1836. {
  1837. struct file *file = iocb->ki_filp;
  1838. struct address_space * mapping = file->f_mapping;
  1839. struct address_space_operations *a_ops = mapping->a_ops;
  1840. struct inode *inode = mapping->host;
  1841. long status = 0;
  1842. struct page *page;
  1843. struct page *cached_page = NULL;
  1844. size_t bytes;
  1845. struct pagevec lru_pvec;
  1846. const struct iovec *cur_iov = iov; /* current iovec */
  1847. size_t iov_base = 0; /* offset in the current iovec */
  1848. char __user *buf;
  1849. pagevec_init(&lru_pvec, 0);
  1850. /*
  1851. * handle partial DIO write. Adjust cur_iov if needed.
  1852. */
  1853. if (likely(nr_segs == 1))
  1854. buf = iov->iov_base + written;
  1855. else {
  1856. filemap_set_next_iovec(&cur_iov, &iov_base, written);
  1857. buf = cur_iov->iov_base + iov_base;
  1858. }
  1859. do {
  1860. unsigned long index;
  1861. unsigned long offset;
  1862. unsigned long maxlen;
  1863. size_t copied;
  1864. offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */
  1865. index = pos >> PAGE_CACHE_SHIFT;
  1866. bytes = PAGE_CACHE_SIZE - offset;
  1867. if (bytes > count)
  1868. bytes = count;
  1869. /*
  1870. * Bring in the user page that we will copy from _first_.
  1871. * Otherwise there's a nasty deadlock on copying from the
  1872. * same page as we're writing to, without it being marked
  1873. * up-to-date.
  1874. */
  1875. maxlen = cur_iov->iov_len - iov_base;
  1876. if (maxlen > bytes)
  1877. maxlen = bytes;
  1878. fault_in_pages_readable(buf, maxlen);
  1879. page = __grab_cache_page(mapping,index,&cached_page,&lru_pvec);
  1880. if (!page) {
  1881. status = -ENOMEM;
  1882. break;
  1883. }
  1884. status = a_ops->prepare_write(file, page, offset, offset+bytes);
  1885. if (unlikely(status)) {
  1886. loff_t isize = i_size_read(inode);
  1887. if (status != AOP_TRUNCATED_PAGE)
  1888. unlock_page(page);
  1889. page_cache_release(page);
  1890. if (status == AOP_TRUNCATED_PAGE)
  1891. continue;
  1892. /*
  1893. * prepare_write() may have instantiated a few blocks
  1894. * outside i_size. Trim these off again.
  1895. */
  1896. if (pos + bytes > isize)
  1897. vmtruncate(inode, isize);
  1898. break;
  1899. }
  1900. if (likely(nr_segs == 1))
  1901. copied = filemap_copy_from_user(page, offset,
  1902. buf, bytes);
  1903. else
  1904. copied = filemap_copy_from_user_iovec(page, offset,
  1905. cur_iov, iov_base, bytes);
  1906. flush_dcache_page(page);
  1907. status = a_ops->commit_write(file, page, offset, offset+bytes);
  1908. if (status == AOP_TRUNCATED_PAGE) {
  1909. page_cache_release(page);
  1910. continue;
  1911. }
  1912. if (likely(copied > 0)) {
  1913. if (!status)
  1914. status = copied;
  1915. if (status >= 0) {
  1916. written += status;
  1917. count -= status;
  1918. pos += status;
  1919. buf += status;
  1920. if (unlikely(nr_segs > 1)) {
  1921. filemap_set_next_iovec(&cur_iov,
  1922. &iov_base, status);
  1923. if (count)
  1924. buf = cur_iov->iov_base +
  1925. iov_base;
  1926. } else {
  1927. iov_base += status;
  1928. }
  1929. }
  1930. }
  1931. if (unlikely(copied != bytes))
  1932. if (status >= 0)
  1933. status = -EFAULT;
  1934. unlock_page(page);
  1935. mark_page_accessed(page);
  1936. page_cache_release(page);
  1937. if (status < 0)
  1938. break;
  1939. balance_dirty_pages_ratelimited(mapping);
  1940. cond_resched();
  1941. } while (count);
  1942. *ppos = pos;
  1943. if (cached_page)
  1944. page_cache_release(cached_page);
  1945. /*
  1946. * For now, when the user asks for O_SYNC, we'll actually give O_DSYNC
  1947. */
  1948. if (likely(status >= 0)) {
  1949. if (unlikely((file->f_flags & O_SYNC) || IS_SYNC(inode))) {
  1950. if (!a_ops->writepage || !is_sync_kiocb(iocb))
  1951. status = generic_osync_inode(inode, mapping,
  1952. OSYNC_METADATA|OSYNC_DATA);
  1953. }
  1954. }
  1955. /*
  1956. * If we get here for O_DIRECT writes then we must have fallen through
  1957. * to buffered writes (block instantiation inside i_size). So we sync
  1958. * the file data here, to try to honour O_DIRECT expectations.
  1959. */
  1960. if (unlikely(file->f_flags & O_DIRECT) && written)
  1961. status = filemap_write_and_wait(mapping);
  1962. pagevec_lru_add(&lru_pvec);
  1963. return written ? written : status;
  1964. }
  1965. EXPORT_SYMBOL(generic_file_buffered_write);
  1966. static ssize_t
  1967. __generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov,
  1968. unsigned long nr_segs, loff_t *ppos)
  1969. {
  1970. struct file *file = iocb->ki_filp;
  1971. struct address_space * mapping = file->f_mapping;
  1972. size_t ocount; /* original count */
  1973. size_t count; /* after file limit checks */
  1974. struct inode *inode = mapping->host;
  1975. unsigned long seg;
  1976. loff_t pos;
  1977. ssize_t written;
  1978. ssize_t err;
  1979. ocount = 0;
  1980. for (seg = 0; seg < nr_segs; seg++) {
  1981. const struct iovec *iv = &iov[seg];
  1982. /*
  1983. * If any segment has a negative length, or the cumulative
  1984. * length ever wraps negative then return -EINVAL.
  1985. */
  1986. ocount += iv->iov_len;
  1987. if (unlikely((ssize_t)(ocount|iv->iov_len) < 0))
  1988. return -EINVAL;
  1989. if (access_ok(VERIFY_READ, iv->iov_base, iv->iov_len))
  1990. continue;
  1991. if (seg == 0)
  1992. return -EFAULT;
  1993. nr_segs = seg;
  1994. ocount -= iv->iov_len; /* This segment is no good */
  1995. break;
  1996. }
  1997. count = ocount;
  1998. pos = *ppos;
  1999. vfs_check_frozen(inode->i_sb, SB_FREEZE_WRITE);
  2000. /* We can write back this queue in page reclaim */
  2001. current->backing_dev_info = mapping->backing_dev_info;
  2002. written = 0;
  2003. err = generic_write_checks(file, &pos, &count, S_ISBLK(inode->i_mode));
  2004. if (err)
  2005. goto out;
  2006. if (count == 0)
  2007. goto out;
  2008. err = remove_suid(file->f_dentry);
  2009. if (err)
  2010. goto out;
  2011. file_update_time(file);
  2012. /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
  2013. if (unlikely(file->f_flags & O_DIRECT)) {
  2014. written = generic_file_direct_write(iocb, iov,
  2015. &nr_segs, pos, ppos, count, ocount);
  2016. if (written < 0 || written == count)
  2017. goto out;
  2018. /*
  2019. * direct-io write to a hole: fall through to buffered I/O
  2020. * for completing the rest of the request.
  2021. */
  2022. pos += written;
  2023. count -= written;
  2024. }
  2025. written = generic_file_buffered_write(iocb, iov, nr_segs,
  2026. pos, ppos, count, written);
  2027. out:
  2028. current->backing_dev_info = NULL;
  2029. return written ? written : err;
  2030. }
  2031. EXPORT_SYMBOL(generic_file_aio_write_nolock);
  2032. ssize_t
  2033. generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov,
  2034. unsigned long nr_segs, loff_t *ppos)
  2035. {
  2036. struct file *file = iocb->ki_filp;
  2037. struct address_space *mapping = file->f_mapping;
  2038. struct inode *inode = mapping->host;
  2039. ssize_t ret;
  2040. loff_t pos = *ppos;
  2041. ret = __generic_file_aio_write_nolock(iocb, iov, nr_segs, ppos);
  2042. if (ret > 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) {
  2043. int err;
  2044. err = sync_page_range_nolock(inode, mapping, pos, ret);
  2045. if (err < 0)
  2046. ret = err;
  2047. }
  2048. return ret;
  2049. }
  2050. static ssize_t
  2051. __generic_file_write_nolock(struct file *file, const struct iovec *iov,
  2052. unsigned long nr_segs, loff_t *ppos)
  2053. {
  2054. struct kiocb kiocb;
  2055. ssize_t ret;
  2056. init_sync_kiocb(&kiocb, file);
  2057. ret = __generic_file_aio_write_nolock(&kiocb, iov, nr_segs, ppos);
  2058. if (ret == -EIOCBQUEUED)
  2059. ret = wait_on_sync_kiocb(&kiocb);
  2060. return ret;
  2061. }
  2062. ssize_t
  2063. generic_file_write_nolock(struct file *file, const struct iovec *iov,
  2064. unsigned long nr_segs, loff_t *ppos)
  2065. {
  2066. struct kiocb kiocb;
  2067. ssize_t ret;
  2068. init_sync_kiocb(&kiocb, file);
  2069. ret = generic_file_aio_write_nolock(&kiocb, iov, nr_segs, ppos);
  2070. if (-EIOCBQUEUED == ret)
  2071. ret = wait_on_sync_kiocb(&kiocb);
  2072. return ret;
  2073. }
  2074. EXPORT_SYMBOL(generic_file_write_nolock);
  2075. ssize_t generic_file_aio_write(struct kiocb *iocb, const char __user *buf,
  2076. size_t count, loff_t pos)
  2077. {
  2078. struct file *file = iocb->ki_filp;
  2079. struct address_space *mapping = file->f_mapping;
  2080. struct inode *inode = mapping->host;
  2081. ssize_t ret;
  2082. struct iovec local_iov = { .iov_base = (void __user *)buf,
  2083. .iov_len = count };
  2084. BUG_ON(iocb->ki_pos != pos);
  2085. mutex_lock(&inode->i_mutex);
  2086. ret = __generic_file_aio_write_nolock(iocb, &local_iov, 1,
  2087. &iocb->ki_pos);
  2088. mutex_unlock(&inode->i_mutex);
  2089. if (ret > 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) {
  2090. ssize_t err;
  2091. err = sync_page_range(inode, mapping, pos, ret);
  2092. if (err < 0)
  2093. ret = err;
  2094. }
  2095. return ret;
  2096. }
  2097. EXPORT_SYMBOL(generic_file_aio_write);
  2098. ssize_t generic_file_write(struct file *file, const char __user *buf,
  2099. size_t count, loff_t *ppos)
  2100. {
  2101. struct address_space *mapping = file->f_mapping;
  2102. struct inode *inode = mapping->host;
  2103. ssize_t ret;
  2104. struct iovec local_iov = { .iov_base = (void __user *)buf,
  2105. .iov_len = count };
  2106. mutex_lock(&inode->i_mutex);
  2107. ret = __generic_file_write_nolock(file, &local_iov, 1, ppos);
  2108. mutex_unlock(&inode->i_mutex);
  2109. if (ret > 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) {
  2110. ssize_t err;
  2111. err = sync_page_range(inode, mapping, *ppos - ret, ret);
  2112. if (err < 0)
  2113. ret = err;
  2114. }
  2115. return ret;
  2116. }
  2117. EXPORT_SYMBOL(generic_file_write);
  2118. ssize_t generic_file_readv(struct file *filp, const struct iovec *iov,
  2119. unsigned long nr_segs, loff_t *ppos)
  2120. {
  2121. struct kiocb kiocb;
  2122. ssize_t ret;
  2123. init_sync_kiocb(&kiocb, filp);
  2124. ret = __generic_file_aio_read(&kiocb, iov, nr_segs, ppos);
  2125. if (-EIOCBQUEUED == ret)
  2126. ret = wait_on_sync_kiocb(&kiocb);
  2127. return ret;
  2128. }
  2129. EXPORT_SYMBOL(generic_file_readv);
  2130. ssize_t generic_file_writev(struct file *file, const struct iovec *iov,
  2131. unsigned long nr_segs, loff_t *ppos)
  2132. {
  2133. struct address_space *mapping = file->f_mapping;
  2134. struct inode *inode = mapping->host;
  2135. ssize_t ret;
  2136. mutex_lock(&inode->i_mutex);
  2137. ret = __generic_file_write_nolock(file, iov, nr_segs, ppos);
  2138. mutex_unlock(&inode->i_mutex);
  2139. if (ret > 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) {
  2140. int err;
  2141. err = sync_page_range(inode, mapping, *ppos - ret, ret);
  2142. if (err < 0)
  2143. ret = err;
  2144. }
  2145. return ret;
  2146. }
  2147. EXPORT_SYMBOL(generic_file_writev);
  2148. /*
  2149. * Called under i_mutex for writes to S_ISREG files. Returns -EIO if something
  2150. * went wrong during pagecache shootdown.
  2151. */
  2152. static ssize_t
  2153. generic_file_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
  2154. loff_t offset, unsigned long nr_segs)
  2155. {
  2156. struct file *file = iocb->ki_filp;
  2157. struct address_space *mapping = file->f_mapping;
  2158. ssize_t retval;
  2159. size_t write_len = 0;
  2160. /*
  2161. * If it's a write, unmap all mmappings of the file up-front. This
  2162. * will cause any pte dirty bits to be propagated into the pageframes
  2163. * for the subsequent filemap_write_and_wait().
  2164. */
  2165. if (rw == WRITE) {
  2166. write_len = iov_length(iov, nr_segs);
  2167. if (mapping_mapped(mapping))
  2168. unmap_mapping_range(mapping, offset, write_len, 0);
  2169. }
  2170. retval = filemap_write_and_wait(mapping);
  2171. if (retval == 0) {
  2172. retval = mapping->a_ops->direct_IO(rw, iocb, iov,
  2173. offset, nr_segs);
  2174. if (rw == WRITE && mapping->nrpages) {
  2175. pgoff_t end = (offset + write_len - 1)
  2176. >> PAGE_CACHE_SHIFT;
  2177. int err = invalidate_inode_pages2_range(mapping,
  2178. offset >> PAGE_CACHE_SHIFT, end);
  2179. if (err)
  2180. retval = err;
  2181. }
  2182. }
  2183. return retval;
  2184. }