filemap.c 68 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596
  1. /*
  2. * linux/mm/filemap.c
  3. *
  4. * Copyright (C) 1994-1999 Linus Torvalds
  5. */
  6. /*
  7. * This file handles the generic file mmap semantics used by
  8. * most "normal" filesystems (but you don't /have/ to use this:
  9. * the NFS filesystem used to do this differently, for example)
  10. */
  11. #include <linux/export.h>
  12. #include <linux/compiler.h>
  13. #include <linux/fs.h>
  14. #include <linux/uaccess.h>
  15. #include <linux/aio.h>
  16. #include <linux/capability.h>
  17. #include <linux/kernel_stat.h>
  18. #include <linux/gfp.h>
  19. #include <linux/mm.h>
  20. #include <linux/swap.h>
  21. #include <linux/mman.h>
  22. #include <linux/pagemap.h>
  23. #include <linux/file.h>
  24. #include <linux/uio.h>
  25. #include <linux/hash.h>
  26. #include <linux/writeback.h>
  27. #include <linux/backing-dev.h>
  28. #include <linux/pagevec.h>
  29. #include <linux/blkdev.h>
  30. #include <linux/security.h>
  31. #include <linux/cpuset.h>
  32. #include <linux/hardirq.h> /* for BUG_ON(!in_atomic()) only */
  33. #include <linux/memcontrol.h>
  34. #include <linux/cleancache.h>
  35. #include "internal.h"
  36. #define CREATE_TRACE_POINTS
  37. #include <trace/events/filemap.h>
  38. /*
  39. * FIXME: remove all knowledge of the buffer layer from the core VM
  40. */
  41. #include <linux/buffer_head.h> /* for try_to_free_buffers */
  42. #include <asm/mman.h>
  43. /*
  44. * Shared mappings implemented 30.11.1994. It's not fully working yet,
  45. * though.
  46. *
  47. * Shared mappings now work. 15.8.1995 Bruno.
  48. *
  49. * finished 'unifying' the page and buffer cache and SMP-threaded the
  50. * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
  51. *
  52. * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
  53. */
  54. /*
  55. * Lock ordering:
  56. *
  57. * ->i_mmap_mutex (truncate_pagecache)
  58. * ->private_lock (__free_pte->__set_page_dirty_buffers)
  59. * ->swap_lock (exclusive_swap_page, others)
  60. * ->mapping->tree_lock
  61. *
  62. * ->i_mutex
  63. * ->i_mmap_mutex (truncate->unmap_mapping_range)
  64. *
  65. * ->mmap_sem
  66. * ->i_mmap_mutex
  67. * ->page_table_lock or pte_lock (various, mainly in memory.c)
  68. * ->mapping->tree_lock (arch-dependent flush_dcache_mmap_lock)
  69. *
  70. * ->mmap_sem
  71. * ->lock_page (access_process_vm)
  72. *
  73. * ->i_mutex (generic_file_buffered_write)
  74. * ->mmap_sem (fault_in_pages_readable->do_page_fault)
  75. *
  76. * bdi->wb.list_lock
  77. * sb_lock (fs/fs-writeback.c)
  78. * ->mapping->tree_lock (__sync_single_inode)
  79. *
  80. * ->i_mmap_mutex
  81. * ->anon_vma.lock (vma_adjust)
  82. *
  83. * ->anon_vma.lock
  84. * ->page_table_lock or pte_lock (anon_vma_prepare and various)
  85. *
  86. * ->page_table_lock or pte_lock
  87. * ->swap_lock (try_to_unmap_one)
  88. * ->private_lock (try_to_unmap_one)
  89. * ->tree_lock (try_to_unmap_one)
  90. * ->zone.lru_lock (follow_page->mark_page_accessed)
  91. * ->zone.lru_lock (check_pte_range->isolate_lru_page)
  92. * ->private_lock (page_remove_rmap->set_page_dirty)
  93. * ->tree_lock (page_remove_rmap->set_page_dirty)
  94. * bdi.wb->list_lock (page_remove_rmap->set_page_dirty)
  95. * ->inode->i_lock (page_remove_rmap->set_page_dirty)
  96. * bdi.wb->list_lock (zap_pte_range->set_page_dirty)
  97. * ->inode->i_lock (zap_pte_range->set_page_dirty)
  98. * ->private_lock (zap_pte_range->__set_page_dirty_buffers)
  99. *
  100. * ->i_mmap_mutex
  101. * ->tasklist_lock (memory_failure, collect_procs_ao)
  102. */
  103. /*
  104. * Delete a page from the page cache and free it. Caller has to make
  105. * sure the page is locked and that nobody else uses it - or that usage
  106. * is safe. The caller must hold the mapping's tree_lock.
  107. */
  108. void __delete_from_page_cache(struct page *page)
  109. {
  110. struct address_space *mapping = page->mapping;
  111. trace_mm_filemap_delete_from_page_cache(page);
  112. /*
  113. * if we're uptodate, flush out into the cleancache, otherwise
  114. * invalidate any existing cleancache entries. We can't leave
  115. * stale data around in the cleancache once our page is gone
  116. */
  117. if (PageUptodate(page) && PageMappedToDisk(page))
  118. cleancache_put_page(page);
  119. else
  120. cleancache_invalidate_page(mapping, page);
  121. radix_tree_delete(&mapping->page_tree, page->index);
  122. page->mapping = NULL;
  123. /* Leave page->index set: truncation lookup relies upon it */
  124. mapping->nrpages--;
  125. __dec_zone_page_state(page, NR_FILE_PAGES);
  126. if (PageSwapBacked(page))
  127. __dec_zone_page_state(page, NR_SHMEM);
  128. BUG_ON(page_mapped(page));
  129. /*
  130. * Some filesystems seem to re-dirty the page even after
  131. * the VM has canceled the dirty bit (eg ext3 journaling).
  132. *
  133. * Fix it up by doing a final dirty accounting check after
  134. * having removed the page entirely.
  135. */
  136. if (PageDirty(page) && mapping_cap_account_dirty(mapping)) {
  137. dec_zone_page_state(page, NR_FILE_DIRTY);
  138. dec_bdi_stat(mapping->backing_dev_info, BDI_RECLAIMABLE);
  139. }
  140. }
  141. /**
  142. * delete_from_page_cache - delete page from page cache
  143. * @page: the page which the kernel is trying to remove from page cache
  144. *
  145. * This must be called only on pages that have been verified to be in the page
  146. * cache and locked. It will never put the page into the free list, the caller
  147. * has a reference on the page.
  148. */
  149. void delete_from_page_cache(struct page *page)
  150. {
  151. struct address_space *mapping = page->mapping;
  152. void (*freepage)(struct page *);
  153. BUG_ON(!PageLocked(page));
  154. freepage = mapping->a_ops->freepage;
  155. spin_lock_irq(&mapping->tree_lock);
  156. __delete_from_page_cache(page);
  157. spin_unlock_irq(&mapping->tree_lock);
  158. mem_cgroup_uncharge_cache_page(page);
  159. if (freepage)
  160. freepage(page);
  161. page_cache_release(page);
  162. }
  163. EXPORT_SYMBOL(delete_from_page_cache);
  164. static int sleep_on_page(void *word)
  165. {
  166. io_schedule();
  167. return 0;
  168. }
  169. static int sleep_on_page_killable(void *word)
  170. {
  171. sleep_on_page(word);
  172. return fatal_signal_pending(current) ? -EINTR : 0;
  173. }
  174. static int filemap_check_errors(struct address_space *mapping)
  175. {
  176. int ret = 0;
  177. /* Check for outstanding write errors */
  178. if (test_and_clear_bit(AS_ENOSPC, &mapping->flags))
  179. ret = -ENOSPC;
  180. if (test_and_clear_bit(AS_EIO, &mapping->flags))
  181. ret = -EIO;
  182. return ret;
  183. }
  184. /**
  185. * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
  186. * @mapping: address space structure to write
  187. * @start: offset in bytes where the range starts
  188. * @end: offset in bytes where the range ends (inclusive)
  189. * @sync_mode: enable synchronous operation
  190. *
  191. * Start writeback against all of a mapping's dirty pages that lie
  192. * within the byte offsets <start, end> inclusive.
  193. *
  194. * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
  195. * opposed to a regular memory cleansing writeback. The difference between
  196. * these two operations is that if a dirty page/buffer is encountered, it must
  197. * be waited upon, and not just skipped over.
  198. */
  199. int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
  200. loff_t end, int sync_mode)
  201. {
  202. int ret;
  203. struct writeback_control wbc = {
  204. .sync_mode = sync_mode,
  205. .nr_to_write = LONG_MAX,
  206. .range_start = start,
  207. .range_end = end,
  208. };
  209. if (!mapping_cap_writeback_dirty(mapping))
  210. return 0;
  211. ret = do_writepages(mapping, &wbc);
  212. return ret;
  213. }
  214. static inline int __filemap_fdatawrite(struct address_space *mapping,
  215. int sync_mode)
  216. {
  217. return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);
  218. }
  219. int filemap_fdatawrite(struct address_space *mapping)
  220. {
  221. return __filemap_fdatawrite(mapping, WB_SYNC_ALL);
  222. }
  223. EXPORT_SYMBOL(filemap_fdatawrite);
  224. int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
  225. loff_t end)
  226. {
  227. return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);
  228. }
  229. EXPORT_SYMBOL(filemap_fdatawrite_range);
  230. /**
  231. * filemap_flush - mostly a non-blocking flush
  232. * @mapping: target address_space
  233. *
  234. * This is a mostly non-blocking flush. Not suitable for data-integrity
  235. * purposes - I/O may not be started against all dirty pages.
  236. */
  237. int filemap_flush(struct address_space *mapping)
  238. {
  239. return __filemap_fdatawrite(mapping, WB_SYNC_NONE);
  240. }
  241. EXPORT_SYMBOL(filemap_flush);
  242. /**
  243. * filemap_fdatawait_range - wait for writeback to complete
  244. * @mapping: address space structure to wait for
  245. * @start_byte: offset in bytes where the range starts
  246. * @end_byte: offset in bytes where the range ends (inclusive)
  247. *
  248. * Walk the list of under-writeback pages of the given address space
  249. * in the given range and wait for all of them.
  250. */
  251. int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte,
  252. loff_t end_byte)
  253. {
  254. pgoff_t index = start_byte >> PAGE_CACHE_SHIFT;
  255. pgoff_t end = end_byte >> PAGE_CACHE_SHIFT;
  256. struct pagevec pvec;
  257. int nr_pages;
  258. int ret2, ret = 0;
  259. if (end_byte < start_byte)
  260. goto out;
  261. pagevec_init(&pvec, 0);
  262. while ((index <= end) &&
  263. (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
  264. PAGECACHE_TAG_WRITEBACK,
  265. min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1)) != 0) {
  266. unsigned i;
  267. for (i = 0; i < nr_pages; i++) {
  268. struct page *page = pvec.pages[i];
  269. /* until radix tree lookup accepts end_index */
  270. if (page->index > end)
  271. continue;
  272. wait_on_page_writeback(page);
  273. if (TestClearPageError(page))
  274. ret = -EIO;
  275. }
  276. pagevec_release(&pvec);
  277. cond_resched();
  278. }
  279. out:
  280. ret2 = filemap_check_errors(mapping);
  281. if (!ret)
  282. ret = ret2;
  283. return ret;
  284. }
  285. EXPORT_SYMBOL(filemap_fdatawait_range);
  286. /**
  287. * filemap_fdatawait - wait for all under-writeback pages to complete
  288. * @mapping: address space structure to wait for
  289. *
  290. * Walk the list of under-writeback pages of the given address space
  291. * and wait for all of them.
  292. */
  293. int filemap_fdatawait(struct address_space *mapping)
  294. {
  295. loff_t i_size = i_size_read(mapping->host);
  296. if (i_size == 0)
  297. return 0;
  298. return filemap_fdatawait_range(mapping, 0, i_size - 1);
  299. }
  300. EXPORT_SYMBOL(filemap_fdatawait);
  301. int filemap_write_and_wait(struct address_space *mapping)
  302. {
  303. int err = 0;
  304. if (mapping->nrpages) {
  305. err = filemap_fdatawrite(mapping);
  306. /*
  307. * Even if the above returned error, the pages may be
  308. * written partially (e.g. -ENOSPC), so we wait for it.
  309. * But the -EIO is special case, it may indicate the worst
  310. * thing (e.g. bug) happened, so we avoid waiting for it.
  311. */
  312. if (err != -EIO) {
  313. int err2 = filemap_fdatawait(mapping);
  314. if (!err)
  315. err = err2;
  316. }
  317. } else {
  318. err = filemap_check_errors(mapping);
  319. }
  320. return err;
  321. }
  322. EXPORT_SYMBOL(filemap_write_and_wait);
  323. /**
  324. * filemap_write_and_wait_range - write out & wait on a file range
  325. * @mapping: the address_space for the pages
  326. * @lstart: offset in bytes where the range starts
  327. * @lend: offset in bytes where the range ends (inclusive)
  328. *
  329. * Write out and wait upon file offsets lstart->lend, inclusive.
  330. *
  331. * Note that `lend' is inclusive (describes the last byte to be written) so
  332. * that this function can be used to write to the very end-of-file (end = -1).
  333. */
  334. int filemap_write_and_wait_range(struct address_space *mapping,
  335. loff_t lstart, loff_t lend)
  336. {
  337. int err = 0;
  338. if (mapping->nrpages) {
  339. err = __filemap_fdatawrite_range(mapping, lstart, lend,
  340. WB_SYNC_ALL);
  341. /* See comment of filemap_write_and_wait() */
  342. if (err != -EIO) {
  343. int err2 = filemap_fdatawait_range(mapping,
  344. lstart, lend);
  345. if (!err)
  346. err = err2;
  347. }
  348. } else {
  349. err = filemap_check_errors(mapping);
  350. }
  351. return err;
  352. }
  353. EXPORT_SYMBOL(filemap_write_and_wait_range);
  354. /**
  355. * replace_page_cache_page - replace a pagecache page with a new one
  356. * @old: page to be replaced
  357. * @new: page to replace with
  358. * @gfp_mask: allocation mode
  359. *
  360. * This function replaces a page in the pagecache with a new one. On
  361. * success it acquires the pagecache reference for the new page and
  362. * drops it for the old page. Both the old and new pages must be
  363. * locked. This function does not add the new page to the LRU, the
  364. * caller must do that.
  365. *
  366. * The remove + add is atomic. The only way this function can fail is
  367. * memory allocation failure.
  368. */
  369. int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask)
  370. {
  371. int error;
  372. VM_BUG_ON(!PageLocked(old));
  373. VM_BUG_ON(!PageLocked(new));
  374. VM_BUG_ON(new->mapping);
  375. error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM);
  376. if (!error) {
  377. struct address_space *mapping = old->mapping;
  378. void (*freepage)(struct page *);
  379. pgoff_t offset = old->index;
  380. freepage = mapping->a_ops->freepage;
  381. page_cache_get(new);
  382. new->mapping = mapping;
  383. new->index = offset;
  384. spin_lock_irq(&mapping->tree_lock);
  385. __delete_from_page_cache(old);
  386. error = radix_tree_insert(&mapping->page_tree, offset, new);
  387. BUG_ON(error);
  388. mapping->nrpages++;
  389. __inc_zone_page_state(new, NR_FILE_PAGES);
  390. if (PageSwapBacked(new))
  391. __inc_zone_page_state(new, NR_SHMEM);
  392. spin_unlock_irq(&mapping->tree_lock);
  393. /* mem_cgroup codes must not be called under tree_lock */
  394. mem_cgroup_replace_page_cache(old, new);
  395. radix_tree_preload_end();
  396. if (freepage)
  397. freepage(old);
  398. page_cache_release(old);
  399. }
  400. return error;
  401. }
  402. EXPORT_SYMBOL_GPL(replace_page_cache_page);
  403. /**
  404. * add_to_page_cache_locked - add a locked page to the pagecache
  405. * @page: page to add
  406. * @mapping: the page's address_space
  407. * @offset: page index
  408. * @gfp_mask: page allocation mode
  409. *
  410. * This function is used to add a page to the pagecache. It must be locked.
  411. * This function does not add the page to the LRU. The caller must do that.
  412. */
  413. int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
  414. pgoff_t offset, gfp_t gfp_mask)
  415. {
  416. int error;
  417. VM_BUG_ON(!PageLocked(page));
  418. VM_BUG_ON(PageSwapBacked(page));
  419. error = mem_cgroup_cache_charge(page, current->mm,
  420. gfp_mask & GFP_RECLAIM_MASK);
  421. if (error)
  422. return error;
  423. error = radix_tree_maybe_preload(gfp_mask & ~__GFP_HIGHMEM);
  424. if (error) {
  425. mem_cgroup_uncharge_cache_page(page);
  426. return error;
  427. }
  428. page_cache_get(page);
  429. page->mapping = mapping;
  430. page->index = offset;
  431. spin_lock_irq(&mapping->tree_lock);
  432. error = radix_tree_insert(&mapping->page_tree, offset, page);
  433. radix_tree_preload_end();
  434. if (unlikely(error))
  435. goto err_insert;
  436. mapping->nrpages++;
  437. __inc_zone_page_state(page, NR_FILE_PAGES);
  438. spin_unlock_irq(&mapping->tree_lock);
  439. trace_mm_filemap_add_to_page_cache(page);
  440. return 0;
  441. err_insert:
  442. page->mapping = NULL;
  443. /* Leave page->index set: truncation relies upon it */
  444. spin_unlock_irq(&mapping->tree_lock);
  445. mem_cgroup_uncharge_cache_page(page);
  446. page_cache_release(page);
  447. return error;
  448. }
  449. EXPORT_SYMBOL(add_to_page_cache_locked);
  450. int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
  451. pgoff_t offset, gfp_t gfp_mask)
  452. {
  453. int ret;
  454. ret = add_to_page_cache(page, mapping, offset, gfp_mask);
  455. if (ret == 0)
  456. lru_cache_add_file(page);
  457. return ret;
  458. }
  459. EXPORT_SYMBOL_GPL(add_to_page_cache_lru);
  460. #ifdef CONFIG_NUMA
  461. struct page *__page_cache_alloc(gfp_t gfp)
  462. {
  463. int n;
  464. struct page *page;
  465. if (cpuset_do_page_mem_spread()) {
  466. unsigned int cpuset_mems_cookie;
  467. do {
  468. cpuset_mems_cookie = get_mems_allowed();
  469. n = cpuset_mem_spread_node();
  470. page = alloc_pages_exact_node(n, gfp, 0);
  471. } while (!put_mems_allowed(cpuset_mems_cookie) && !page);
  472. return page;
  473. }
  474. return alloc_pages(gfp, 0);
  475. }
  476. EXPORT_SYMBOL(__page_cache_alloc);
  477. #endif
  478. /*
  479. * In order to wait for pages to become available there must be
  480. * waitqueues associated with pages. By using a hash table of
  481. * waitqueues where the bucket discipline is to maintain all
  482. * waiters on the same queue and wake all when any of the pages
  483. * become available, and for the woken contexts to check to be
  484. * sure the appropriate page became available, this saves space
  485. * at a cost of "thundering herd" phenomena during rare hash
  486. * collisions.
  487. */
  488. static wait_queue_head_t *page_waitqueue(struct page *page)
  489. {
  490. const struct zone *zone = page_zone(page);
  491. return &zone->wait_table[hash_ptr(page, zone->wait_table_bits)];
  492. }
  493. static inline void wake_up_page(struct page *page, int bit)
  494. {
  495. __wake_up_bit(page_waitqueue(page), &page->flags, bit);
  496. }
  497. void wait_on_page_bit(struct page *page, int bit_nr)
  498. {
  499. DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);
  500. if (test_bit(bit_nr, &page->flags))
  501. __wait_on_bit(page_waitqueue(page), &wait, sleep_on_page,
  502. TASK_UNINTERRUPTIBLE);
  503. }
  504. EXPORT_SYMBOL(wait_on_page_bit);
  505. int wait_on_page_bit_killable(struct page *page, int bit_nr)
  506. {
  507. DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);
  508. if (!test_bit(bit_nr, &page->flags))
  509. return 0;
  510. return __wait_on_bit(page_waitqueue(page), &wait,
  511. sleep_on_page_killable, TASK_KILLABLE);
  512. }
  513. /**
  514. * add_page_wait_queue - Add an arbitrary waiter to a page's wait queue
  515. * @page: Page defining the wait queue of interest
  516. * @waiter: Waiter to add to the queue
  517. *
  518. * Add an arbitrary @waiter to the wait queue for the nominated @page.
  519. */
  520. void add_page_wait_queue(struct page *page, wait_queue_t *waiter)
  521. {
  522. wait_queue_head_t *q = page_waitqueue(page);
  523. unsigned long flags;
  524. spin_lock_irqsave(&q->lock, flags);
  525. __add_wait_queue(q, waiter);
  526. spin_unlock_irqrestore(&q->lock, flags);
  527. }
  528. EXPORT_SYMBOL_GPL(add_page_wait_queue);
  529. /**
  530. * unlock_page - unlock a locked page
  531. * @page: the page
  532. *
  533. * Unlocks the page and wakes up sleepers in ___wait_on_page_locked().
  534. * Also wakes sleepers in wait_on_page_writeback() because the wakeup
  535. * mechananism between PageLocked pages and PageWriteback pages is shared.
  536. * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
  537. *
  538. * The mb is necessary to enforce ordering between the clear_bit and the read
  539. * of the waitqueue (to avoid SMP races with a parallel wait_on_page_locked()).
  540. */
  541. void unlock_page(struct page *page)
  542. {
  543. VM_BUG_ON(!PageLocked(page));
  544. clear_bit_unlock(PG_locked, &page->flags);
  545. smp_mb__after_clear_bit();
  546. wake_up_page(page, PG_locked);
  547. }
  548. EXPORT_SYMBOL(unlock_page);
  549. /**
  550. * end_page_writeback - end writeback against a page
  551. * @page: the page
  552. */
  553. void end_page_writeback(struct page *page)
  554. {
  555. if (TestClearPageReclaim(page))
  556. rotate_reclaimable_page(page);
  557. if (!test_clear_page_writeback(page))
  558. BUG();
  559. smp_mb__after_clear_bit();
  560. wake_up_page(page, PG_writeback);
  561. }
  562. EXPORT_SYMBOL(end_page_writeback);
  563. /**
  564. * __lock_page - get a lock on the page, assuming we need to sleep to get it
  565. * @page: the page to lock
  566. */
  567. void __lock_page(struct page *page)
  568. {
  569. DEFINE_WAIT_BIT(wait, &page->flags, PG_locked);
  570. __wait_on_bit_lock(page_waitqueue(page), &wait, sleep_on_page,
  571. TASK_UNINTERRUPTIBLE);
  572. }
  573. EXPORT_SYMBOL(__lock_page);
  574. int __lock_page_killable(struct page *page)
  575. {
  576. DEFINE_WAIT_BIT(wait, &page->flags, PG_locked);
  577. return __wait_on_bit_lock(page_waitqueue(page), &wait,
  578. sleep_on_page_killable, TASK_KILLABLE);
  579. }
  580. EXPORT_SYMBOL_GPL(__lock_page_killable);
  581. int __lock_page_or_retry(struct page *page, struct mm_struct *mm,
  582. unsigned int flags)
  583. {
  584. if (flags & FAULT_FLAG_ALLOW_RETRY) {
  585. /*
  586. * CAUTION! In this case, mmap_sem is not released
  587. * even though return 0.
  588. */
  589. if (flags & FAULT_FLAG_RETRY_NOWAIT)
  590. return 0;
  591. up_read(&mm->mmap_sem);
  592. if (flags & FAULT_FLAG_KILLABLE)
  593. wait_on_page_locked_killable(page);
  594. else
  595. wait_on_page_locked(page);
  596. return 0;
  597. } else {
  598. if (flags & FAULT_FLAG_KILLABLE) {
  599. int ret;
  600. ret = __lock_page_killable(page);
  601. if (ret) {
  602. up_read(&mm->mmap_sem);
  603. return 0;
  604. }
  605. } else
  606. __lock_page(page);
  607. return 1;
  608. }
  609. }
  610. /**
  611. * find_get_page - find and get a page reference
  612. * @mapping: the address_space to search
  613. * @offset: the page index
  614. *
  615. * Is there a pagecache struct page at the given (mapping, offset) tuple?
  616. * If yes, increment its refcount and return it; if no, return NULL.
  617. */
  618. struct page *find_get_page(struct address_space *mapping, pgoff_t offset)
  619. {
  620. void **pagep;
  621. struct page *page;
  622. rcu_read_lock();
  623. repeat:
  624. page = NULL;
  625. pagep = radix_tree_lookup_slot(&mapping->page_tree, offset);
  626. if (pagep) {
  627. page = radix_tree_deref_slot(pagep);
  628. if (unlikely(!page))
  629. goto out;
  630. if (radix_tree_exception(page)) {
  631. if (radix_tree_deref_retry(page))
  632. goto repeat;
  633. /*
  634. * Otherwise, shmem/tmpfs must be storing a swap entry
  635. * here as an exceptional entry: so return it without
  636. * attempting to raise page count.
  637. */
  638. goto out;
  639. }
  640. if (!page_cache_get_speculative(page))
  641. goto repeat;
  642. /*
  643. * Has the page moved?
  644. * This is part of the lockless pagecache protocol. See
  645. * include/linux/pagemap.h for details.
  646. */
  647. if (unlikely(page != *pagep)) {
  648. page_cache_release(page);
  649. goto repeat;
  650. }
  651. }
  652. out:
  653. rcu_read_unlock();
  654. return page;
  655. }
  656. EXPORT_SYMBOL(find_get_page);
  657. /**
  658. * find_lock_page - locate, pin and lock a pagecache page
  659. * @mapping: the address_space to search
  660. * @offset: the page index
  661. *
  662. * Locates the desired pagecache page, locks it, increments its reference
  663. * count and returns its address.
  664. *
  665. * Returns zero if the page was not present. find_lock_page() may sleep.
  666. */
  667. struct page *find_lock_page(struct address_space *mapping, pgoff_t offset)
  668. {
  669. struct page *page;
  670. repeat:
  671. page = find_get_page(mapping, offset);
  672. if (page && !radix_tree_exception(page)) {
  673. lock_page(page);
  674. /* Has the page been truncated? */
  675. if (unlikely(page->mapping != mapping)) {
  676. unlock_page(page);
  677. page_cache_release(page);
  678. goto repeat;
  679. }
  680. VM_BUG_ON(page->index != offset);
  681. }
  682. return page;
  683. }
  684. EXPORT_SYMBOL(find_lock_page);
  685. /**
  686. * find_or_create_page - locate or add a pagecache page
  687. * @mapping: the page's address_space
  688. * @index: the page's index into the mapping
  689. * @gfp_mask: page allocation mode
  690. *
  691. * Locates a page in the pagecache. If the page is not present, a new page
  692. * is allocated using @gfp_mask and is added to the pagecache and to the VM's
  693. * LRU list. The returned page is locked and has its reference count
  694. * incremented.
  695. *
  696. * find_or_create_page() may sleep, even if @gfp_flags specifies an atomic
  697. * allocation!
  698. *
  699. * find_or_create_page() returns the desired page's address, or zero on
  700. * memory exhaustion.
  701. */
  702. struct page *find_or_create_page(struct address_space *mapping,
  703. pgoff_t index, gfp_t gfp_mask)
  704. {
  705. struct page *page;
  706. int err;
  707. repeat:
  708. page = find_lock_page(mapping, index);
  709. if (!page) {
  710. page = __page_cache_alloc(gfp_mask);
  711. if (!page)
  712. return NULL;
  713. /*
  714. * We want a regular kernel memory (not highmem or DMA etc)
  715. * allocation for the radix tree nodes, but we need to honour
  716. * the context-specific requirements the caller has asked for.
  717. * GFP_RECLAIM_MASK collects those requirements.
  718. */
  719. err = add_to_page_cache_lru(page, mapping, index,
  720. (gfp_mask & GFP_RECLAIM_MASK));
  721. if (unlikely(err)) {
  722. page_cache_release(page);
  723. page = NULL;
  724. if (err == -EEXIST)
  725. goto repeat;
  726. }
  727. }
  728. return page;
  729. }
  730. EXPORT_SYMBOL(find_or_create_page);
  731. /**
  732. * find_get_pages - gang pagecache lookup
  733. * @mapping: The address_space to search
  734. * @start: The starting page index
  735. * @nr_pages: The maximum number of pages
  736. * @pages: Where the resulting pages are placed
  737. *
  738. * find_get_pages() will search for and return a group of up to
  739. * @nr_pages pages in the mapping. The pages are placed at @pages.
  740. * find_get_pages() takes a reference against the returned pages.
  741. *
  742. * The search returns a group of mapping-contiguous pages with ascending
  743. * indexes. There may be holes in the indices due to not-present pages.
  744. *
  745. * find_get_pages() returns the number of pages which were found.
  746. */
  747. unsigned find_get_pages(struct address_space *mapping, pgoff_t start,
  748. unsigned int nr_pages, struct page **pages)
  749. {
  750. struct radix_tree_iter iter;
  751. void **slot;
  752. unsigned ret = 0;
  753. if (unlikely(!nr_pages))
  754. return 0;
  755. rcu_read_lock();
  756. restart:
  757. radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
  758. struct page *page;
  759. repeat:
  760. page = radix_tree_deref_slot(slot);
  761. if (unlikely(!page))
  762. continue;
  763. if (radix_tree_exception(page)) {
  764. if (radix_tree_deref_retry(page)) {
  765. /*
  766. * Transient condition which can only trigger
  767. * when entry at index 0 moves out of or back
  768. * to root: none yet gotten, safe to restart.
  769. */
  770. WARN_ON(iter.index);
  771. goto restart;
  772. }
  773. /*
  774. * Otherwise, shmem/tmpfs must be storing a swap entry
  775. * here as an exceptional entry: so skip over it -
  776. * we only reach this from invalidate_mapping_pages().
  777. */
  778. continue;
  779. }
  780. if (!page_cache_get_speculative(page))
  781. goto repeat;
  782. /* Has the page moved? */
  783. if (unlikely(page != *slot)) {
  784. page_cache_release(page);
  785. goto repeat;
  786. }
  787. pages[ret] = page;
  788. if (++ret == nr_pages)
  789. break;
  790. }
  791. rcu_read_unlock();
  792. return ret;
  793. }
  794. /**
  795. * find_get_pages_contig - gang contiguous pagecache lookup
  796. * @mapping: The address_space to search
  797. * @index: The starting page index
  798. * @nr_pages: The maximum number of pages
  799. * @pages: Where the resulting pages are placed
  800. *
  801. * find_get_pages_contig() works exactly like find_get_pages(), except
  802. * that the returned number of pages are guaranteed to be contiguous.
  803. *
  804. * find_get_pages_contig() returns the number of pages which were found.
  805. */
  806. unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
  807. unsigned int nr_pages, struct page **pages)
  808. {
  809. struct radix_tree_iter iter;
  810. void **slot;
  811. unsigned int ret = 0;
  812. if (unlikely(!nr_pages))
  813. return 0;
  814. rcu_read_lock();
  815. restart:
  816. radix_tree_for_each_contig(slot, &mapping->page_tree, &iter, index) {
  817. struct page *page;
  818. repeat:
  819. page = radix_tree_deref_slot(slot);
  820. /* The hole, there no reason to continue */
  821. if (unlikely(!page))
  822. break;
  823. if (radix_tree_exception(page)) {
  824. if (radix_tree_deref_retry(page)) {
  825. /*
  826. * Transient condition which can only trigger
  827. * when entry at index 0 moves out of or back
  828. * to root: none yet gotten, safe to restart.
  829. */
  830. goto restart;
  831. }
  832. /*
  833. * Otherwise, shmem/tmpfs must be storing a swap entry
  834. * here as an exceptional entry: so stop looking for
  835. * contiguous pages.
  836. */
  837. break;
  838. }
  839. if (!page_cache_get_speculative(page))
  840. goto repeat;
  841. /* Has the page moved? */
  842. if (unlikely(page != *slot)) {
  843. page_cache_release(page);
  844. goto repeat;
  845. }
  846. /*
  847. * must check mapping and index after taking the ref.
  848. * otherwise we can get both false positives and false
  849. * negatives, which is just confusing to the caller.
  850. */
  851. if (page->mapping == NULL || page->index != iter.index) {
  852. page_cache_release(page);
  853. break;
  854. }
  855. pages[ret] = page;
  856. if (++ret == nr_pages)
  857. break;
  858. }
  859. rcu_read_unlock();
  860. return ret;
  861. }
  862. EXPORT_SYMBOL(find_get_pages_contig);
  863. /**
  864. * find_get_pages_tag - find and return pages that match @tag
  865. * @mapping: the address_space to search
  866. * @index: the starting page index
  867. * @tag: the tag index
  868. * @nr_pages: the maximum number of pages
  869. * @pages: where the resulting pages are placed
  870. *
  871. * Like find_get_pages, except we only return pages which are tagged with
  872. * @tag. We update @index to index the next page for the traversal.
  873. */
  874. unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index,
  875. int tag, unsigned int nr_pages, struct page **pages)
  876. {
  877. struct radix_tree_iter iter;
  878. void **slot;
  879. unsigned ret = 0;
  880. if (unlikely(!nr_pages))
  881. return 0;
  882. rcu_read_lock();
  883. restart:
  884. radix_tree_for_each_tagged(slot, &mapping->page_tree,
  885. &iter, *index, tag) {
  886. struct page *page;
  887. repeat:
  888. page = radix_tree_deref_slot(slot);
  889. if (unlikely(!page))
  890. continue;
  891. if (radix_tree_exception(page)) {
  892. if (radix_tree_deref_retry(page)) {
  893. /*
  894. * Transient condition which can only trigger
  895. * when entry at index 0 moves out of or back
  896. * to root: none yet gotten, safe to restart.
  897. */
  898. goto restart;
  899. }
  900. /*
  901. * This function is never used on a shmem/tmpfs
  902. * mapping, so a swap entry won't be found here.
  903. */
  904. BUG();
  905. }
  906. if (!page_cache_get_speculative(page))
  907. goto repeat;
  908. /* Has the page moved? */
  909. if (unlikely(page != *slot)) {
  910. page_cache_release(page);
  911. goto repeat;
  912. }
  913. pages[ret] = page;
  914. if (++ret == nr_pages)
  915. break;
  916. }
  917. rcu_read_unlock();
  918. if (ret)
  919. *index = pages[ret - 1]->index + 1;
  920. return ret;
  921. }
  922. EXPORT_SYMBOL(find_get_pages_tag);
  923. /**
  924. * grab_cache_page_nowait - returns locked page at given index in given cache
  925. * @mapping: target address_space
  926. * @index: the page index
  927. *
  928. * Same as grab_cache_page(), but do not wait if the page is unavailable.
  929. * This is intended for speculative data generators, where the data can
  930. * be regenerated if the page couldn't be grabbed. This routine should
  931. * be safe to call while holding the lock for another page.
  932. *
  933. * Clear __GFP_FS when allocating the page to avoid recursion into the fs
  934. * and deadlock against the caller's locked page.
  935. */
  936. struct page *
  937. grab_cache_page_nowait(struct address_space *mapping, pgoff_t index)
  938. {
  939. struct page *page = find_get_page(mapping, index);
  940. if (page) {
  941. if (trylock_page(page))
  942. return page;
  943. page_cache_release(page);
  944. return NULL;
  945. }
  946. page = __page_cache_alloc(mapping_gfp_mask(mapping) & ~__GFP_FS);
  947. if (page && add_to_page_cache_lru(page, mapping, index, GFP_NOFS)) {
  948. page_cache_release(page);
  949. page = NULL;
  950. }
  951. return page;
  952. }
  953. EXPORT_SYMBOL(grab_cache_page_nowait);
  954. /*
  955. * CD/DVDs are error prone. When a medium error occurs, the driver may fail
  956. * a _large_ part of the i/o request. Imagine the worst scenario:
  957. *
  958. * ---R__________________________________________B__________
  959. * ^ reading here ^ bad block(assume 4k)
  960. *
  961. * read(R) => miss => readahead(R...B) => media error => frustrating retries
  962. * => failing the whole request => read(R) => read(R+1) =>
  963. * readahead(R+1...B+1) => bang => read(R+2) => read(R+3) =>
  964. * readahead(R+3...B+2) => bang => read(R+3) => read(R+4) =>
  965. * readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ......
  966. *
  967. * It is going insane. Fix it by quickly scaling down the readahead size.
  968. */
  969. static void shrink_readahead_size_eio(struct file *filp,
  970. struct file_ra_state *ra)
  971. {
  972. ra->ra_pages /= 4;
  973. }
  974. /**
  975. * do_generic_file_read - generic file read routine
  976. * @filp: the file to read
  977. * @ppos: current file position
  978. * @desc: read_descriptor
  979. *
  980. * This is a generic file read routine, and uses the
  981. * mapping->a_ops->readpage() function for the actual low-level stuff.
  982. *
  983. * This is really ugly. But the goto's actually try to clarify some
  984. * of the logic when it comes to error handling etc.
  985. */
  986. static void do_generic_file_read(struct file *filp, loff_t *ppos,
  987. read_descriptor_t *desc)
  988. {
  989. struct address_space *mapping = filp->f_mapping;
  990. struct inode *inode = mapping->host;
  991. struct file_ra_state *ra = &filp->f_ra;
  992. pgoff_t index;
  993. pgoff_t last_index;
  994. pgoff_t prev_index;
  995. unsigned long offset; /* offset into pagecache page */
  996. unsigned int prev_offset;
  997. int error;
  998. index = *ppos >> PAGE_CACHE_SHIFT;
  999. prev_index = ra->prev_pos >> PAGE_CACHE_SHIFT;
  1000. prev_offset = ra->prev_pos & (PAGE_CACHE_SIZE-1);
  1001. last_index = (*ppos + desc->count + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT;
  1002. offset = *ppos & ~PAGE_CACHE_MASK;
  1003. for (;;) {
  1004. struct page *page;
  1005. pgoff_t end_index;
  1006. loff_t isize;
  1007. unsigned long nr, ret;
  1008. cond_resched();
  1009. find_page:
  1010. page = find_get_page(mapping, index);
  1011. if (!page) {
  1012. page_cache_sync_readahead(mapping,
  1013. ra, filp,
  1014. index, last_index - index);
  1015. page = find_get_page(mapping, index);
  1016. if (unlikely(page == NULL))
  1017. goto no_cached_page;
  1018. }
  1019. if (PageReadahead(page)) {
  1020. page_cache_async_readahead(mapping,
  1021. ra, filp, page,
  1022. index, last_index - index);
  1023. }
  1024. if (!PageUptodate(page)) {
  1025. if (inode->i_blkbits == PAGE_CACHE_SHIFT ||
  1026. !mapping->a_ops->is_partially_uptodate)
  1027. goto page_not_up_to_date;
  1028. if (!trylock_page(page))
  1029. goto page_not_up_to_date;
  1030. /* Did it get truncated before we got the lock? */
  1031. if (!page->mapping)
  1032. goto page_not_up_to_date_locked;
  1033. if (!mapping->a_ops->is_partially_uptodate(page,
  1034. desc, offset))
  1035. goto page_not_up_to_date_locked;
  1036. unlock_page(page);
  1037. }
  1038. page_ok:
  1039. /*
  1040. * i_size must be checked after we know the page is Uptodate.
  1041. *
  1042. * Checking i_size after the check allows us to calculate
  1043. * the correct value for "nr", which means the zero-filled
  1044. * part of the page is not copied back to userspace (unless
  1045. * another truncate extends the file - this is desired though).
  1046. */
  1047. isize = i_size_read(inode);
  1048. end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
  1049. if (unlikely(!isize || index > end_index)) {
  1050. page_cache_release(page);
  1051. goto out;
  1052. }
  1053. /* nr is the maximum number of bytes to copy from this page */
  1054. nr = PAGE_CACHE_SIZE;
  1055. if (index == end_index) {
  1056. nr = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
  1057. if (nr <= offset) {
  1058. page_cache_release(page);
  1059. goto out;
  1060. }
  1061. }
  1062. nr = nr - offset;
  1063. /* If users can be writing to this page using arbitrary
  1064. * virtual addresses, take care about potential aliasing
  1065. * before reading the page on the kernel side.
  1066. */
  1067. if (mapping_writably_mapped(mapping))
  1068. flush_dcache_page(page);
  1069. /*
  1070. * When a sequential read accesses a page several times,
  1071. * only mark it as accessed the first time.
  1072. */
  1073. if (prev_index != index || offset != prev_offset)
  1074. mark_page_accessed(page);
  1075. prev_index = index;
  1076. /*
  1077. * Ok, we have the page, and it's up-to-date, so
  1078. * now we can copy it to user space...
  1079. *
  1080. * The file_read_actor routine returns how many bytes were
  1081. * actually used..
  1082. * NOTE! This may not be the same as how much of a user buffer
  1083. * we filled up (we may be padding etc), so we can only update
  1084. * "pos" here (the actor routine has to update the user buffer
  1085. * pointers and the remaining count).
  1086. */
  1087. ret = file_read_actor(desc, page, offset, nr);
  1088. offset += ret;
  1089. index += offset >> PAGE_CACHE_SHIFT;
  1090. offset &= ~PAGE_CACHE_MASK;
  1091. prev_offset = offset;
  1092. page_cache_release(page);
  1093. if (ret == nr && desc->count)
  1094. continue;
  1095. goto out;
  1096. page_not_up_to_date:
  1097. /* Get exclusive access to the page ... */
  1098. error = lock_page_killable(page);
  1099. if (unlikely(error))
  1100. goto readpage_error;
  1101. page_not_up_to_date_locked:
  1102. /* Did it get truncated before we got the lock? */
  1103. if (!page->mapping) {
  1104. unlock_page(page);
  1105. page_cache_release(page);
  1106. continue;
  1107. }
  1108. /* Did somebody else fill it already? */
  1109. if (PageUptodate(page)) {
  1110. unlock_page(page);
  1111. goto page_ok;
  1112. }
  1113. readpage:
  1114. /*
  1115. * A previous I/O error may have been due to temporary
  1116. * failures, eg. multipath errors.
  1117. * PG_error will be set again if readpage fails.
  1118. */
  1119. ClearPageError(page);
  1120. /* Start the actual read. The read will unlock the page. */
  1121. error = mapping->a_ops->readpage(filp, page);
  1122. if (unlikely(error)) {
  1123. if (error == AOP_TRUNCATED_PAGE) {
  1124. page_cache_release(page);
  1125. goto find_page;
  1126. }
  1127. goto readpage_error;
  1128. }
  1129. if (!PageUptodate(page)) {
  1130. error = lock_page_killable(page);
  1131. if (unlikely(error))
  1132. goto readpage_error;
  1133. if (!PageUptodate(page)) {
  1134. if (page->mapping == NULL) {
  1135. /*
  1136. * invalidate_mapping_pages got it
  1137. */
  1138. unlock_page(page);
  1139. page_cache_release(page);
  1140. goto find_page;
  1141. }
  1142. unlock_page(page);
  1143. shrink_readahead_size_eio(filp, ra);
  1144. error = -EIO;
  1145. goto readpage_error;
  1146. }
  1147. unlock_page(page);
  1148. }
  1149. goto page_ok;
  1150. readpage_error:
  1151. /* UHHUH! A synchronous read error occurred. Report it */
  1152. desc->error = error;
  1153. page_cache_release(page);
  1154. goto out;
  1155. no_cached_page:
  1156. /*
  1157. * Ok, it wasn't cached, so we need to create a new
  1158. * page..
  1159. */
  1160. page = page_cache_alloc_cold(mapping);
  1161. if (!page) {
  1162. desc->error = -ENOMEM;
  1163. goto out;
  1164. }
  1165. error = add_to_page_cache_lru(page, mapping,
  1166. index, GFP_KERNEL);
  1167. if (error) {
  1168. page_cache_release(page);
  1169. if (error == -EEXIST)
  1170. goto find_page;
  1171. desc->error = error;
  1172. goto out;
  1173. }
  1174. goto readpage;
  1175. }
  1176. out:
  1177. ra->prev_pos = prev_index;
  1178. ra->prev_pos <<= PAGE_CACHE_SHIFT;
  1179. ra->prev_pos |= prev_offset;
  1180. *ppos = ((loff_t)index << PAGE_CACHE_SHIFT) + offset;
  1181. file_accessed(filp);
  1182. }
  1183. int file_read_actor(read_descriptor_t *desc, struct page *page,
  1184. unsigned long offset, unsigned long size)
  1185. {
  1186. char *kaddr;
  1187. unsigned long left, count = desc->count;
  1188. if (size > count)
  1189. size = count;
  1190. /*
  1191. * Faults on the destination of a read are common, so do it before
  1192. * taking the kmap.
  1193. */
  1194. if (!fault_in_pages_writeable(desc->arg.buf, size)) {
  1195. kaddr = kmap_atomic(page);
  1196. left = __copy_to_user_inatomic(desc->arg.buf,
  1197. kaddr + offset, size);
  1198. kunmap_atomic(kaddr);
  1199. if (left == 0)
  1200. goto success;
  1201. }
  1202. /* Do it the slow way */
  1203. kaddr = kmap(page);
  1204. left = __copy_to_user(desc->arg.buf, kaddr + offset, size);
  1205. kunmap(page);
  1206. if (left) {
  1207. size -= left;
  1208. desc->error = -EFAULT;
  1209. }
  1210. success:
  1211. desc->count = count - size;
  1212. desc->written += size;
  1213. desc->arg.buf += size;
  1214. return size;
  1215. }
  1216. /*
  1217. * Performs necessary checks before doing a write
  1218. * @iov: io vector request
  1219. * @nr_segs: number of segments in the iovec
  1220. * @count: number of bytes to write
  1221. * @access_flags: type of access: %VERIFY_READ or %VERIFY_WRITE
  1222. *
  1223. * Adjust number of segments and amount of bytes to write (nr_segs should be
  1224. * properly initialized first). Returns appropriate error code that caller
  1225. * should return or zero in case that write should be allowed.
  1226. */
  1227. int generic_segment_checks(const struct iovec *iov,
  1228. unsigned long *nr_segs, size_t *count, int access_flags)
  1229. {
  1230. unsigned long seg;
  1231. size_t cnt = 0;
  1232. for (seg = 0; seg < *nr_segs; seg++) {
  1233. const struct iovec *iv = &iov[seg];
  1234. /*
  1235. * If any segment has a negative length, or the cumulative
  1236. * length ever wraps negative then return -EINVAL.
  1237. */
  1238. cnt += iv->iov_len;
  1239. if (unlikely((ssize_t)(cnt|iv->iov_len) < 0))
  1240. return -EINVAL;
  1241. if (access_ok(access_flags, iv->iov_base, iv->iov_len))
  1242. continue;
  1243. if (seg == 0)
  1244. return -EFAULT;
  1245. *nr_segs = seg;
  1246. cnt -= iv->iov_len; /* This segment is no good */
  1247. break;
  1248. }
  1249. *count = cnt;
  1250. return 0;
  1251. }
  1252. EXPORT_SYMBOL(generic_segment_checks);
  1253. /**
  1254. * generic_file_aio_read - generic filesystem read routine
  1255. * @iocb: kernel I/O control block
  1256. * @iov: io vector request
  1257. * @nr_segs: number of segments in the iovec
  1258. * @pos: current file position
  1259. *
  1260. * This is the "read()" routine for all filesystems
  1261. * that can use the page cache directly.
  1262. */
  1263. ssize_t
  1264. generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
  1265. unsigned long nr_segs, loff_t pos)
  1266. {
  1267. struct file *filp = iocb->ki_filp;
  1268. ssize_t retval;
  1269. unsigned long seg = 0;
  1270. size_t count;
  1271. loff_t *ppos = &iocb->ki_pos;
  1272. count = 0;
  1273. retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
  1274. if (retval)
  1275. return retval;
  1276. /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
  1277. if (filp->f_flags & O_DIRECT) {
  1278. loff_t size;
  1279. struct address_space *mapping;
  1280. struct inode *inode;
  1281. mapping = filp->f_mapping;
  1282. inode = mapping->host;
  1283. if (!count)
  1284. goto out; /* skip atime */
  1285. size = i_size_read(inode);
  1286. if (pos < size) {
  1287. retval = filemap_write_and_wait_range(mapping, pos,
  1288. pos + iov_length(iov, nr_segs) - 1);
  1289. if (!retval) {
  1290. retval = mapping->a_ops->direct_IO(READ, iocb,
  1291. iov, pos, nr_segs);
  1292. }
  1293. if (retval > 0) {
  1294. *ppos = pos + retval;
  1295. count -= retval;
  1296. }
  1297. /*
  1298. * Btrfs can have a short DIO read if we encounter
  1299. * compressed extents, so if there was an error, or if
  1300. * we've already read everything we wanted to, or if
  1301. * there was a short read because we hit EOF, go ahead
  1302. * and return. Otherwise fallthrough to buffered io for
  1303. * the rest of the read.
  1304. */
  1305. if (retval < 0 || !count || *ppos >= size) {
  1306. file_accessed(filp);
  1307. goto out;
  1308. }
  1309. }
  1310. }
  1311. count = retval;
  1312. for (seg = 0; seg < nr_segs; seg++) {
  1313. read_descriptor_t desc;
  1314. loff_t offset = 0;
  1315. /*
  1316. * If we did a short DIO read we need to skip the section of the
  1317. * iov that we've already read data into.
  1318. */
  1319. if (count) {
  1320. if (count > iov[seg].iov_len) {
  1321. count -= iov[seg].iov_len;
  1322. continue;
  1323. }
  1324. offset = count;
  1325. count = 0;
  1326. }
  1327. desc.written = 0;
  1328. desc.arg.buf = iov[seg].iov_base + offset;
  1329. desc.count = iov[seg].iov_len - offset;
  1330. if (desc.count == 0)
  1331. continue;
  1332. desc.error = 0;
  1333. do_generic_file_read(filp, ppos, &desc);
  1334. retval += desc.written;
  1335. if (desc.error) {
  1336. retval = retval ?: desc.error;
  1337. break;
  1338. }
  1339. if (desc.count > 0)
  1340. break;
  1341. }
  1342. out:
  1343. return retval;
  1344. }
  1345. EXPORT_SYMBOL(generic_file_aio_read);
  1346. #ifdef CONFIG_MMU
  1347. /**
  1348. * page_cache_read - adds requested page to the page cache if not already there
  1349. * @file: file to read
  1350. * @offset: page index
  1351. *
  1352. * This adds the requested page to the page cache if it isn't already there,
  1353. * and schedules an I/O to read in its contents from disk.
  1354. */
  1355. static int page_cache_read(struct file *file, pgoff_t offset)
  1356. {
  1357. struct address_space *mapping = file->f_mapping;
  1358. struct page *page;
  1359. int ret;
  1360. do {
  1361. page = page_cache_alloc_cold(mapping);
  1362. if (!page)
  1363. return -ENOMEM;
  1364. ret = add_to_page_cache_lru(page, mapping, offset, GFP_KERNEL);
  1365. if (ret == 0)
  1366. ret = mapping->a_ops->readpage(file, page);
  1367. else if (ret == -EEXIST)
  1368. ret = 0; /* losing race to add is OK */
  1369. page_cache_release(page);
  1370. } while (ret == AOP_TRUNCATED_PAGE);
  1371. return ret;
  1372. }
  1373. #define MMAP_LOTSAMISS (100)
  1374. /*
  1375. * Synchronous readahead happens when we don't even find
  1376. * a page in the page cache at all.
  1377. */
  1378. static void do_sync_mmap_readahead(struct vm_area_struct *vma,
  1379. struct file_ra_state *ra,
  1380. struct file *file,
  1381. pgoff_t offset)
  1382. {
  1383. unsigned long ra_pages;
  1384. struct address_space *mapping = file->f_mapping;
  1385. /* If we don't want any read-ahead, don't bother */
  1386. if (vma->vm_flags & VM_RAND_READ)
  1387. return;
  1388. if (!ra->ra_pages)
  1389. return;
  1390. if (vma->vm_flags & VM_SEQ_READ) {
  1391. page_cache_sync_readahead(mapping, ra, file, offset,
  1392. ra->ra_pages);
  1393. return;
  1394. }
  1395. /* Avoid banging the cache line if not needed */
  1396. if (ra->mmap_miss < MMAP_LOTSAMISS * 10)
  1397. ra->mmap_miss++;
  1398. /*
  1399. * Do we miss much more than hit in this file? If so,
  1400. * stop bothering with read-ahead. It will only hurt.
  1401. */
  1402. if (ra->mmap_miss > MMAP_LOTSAMISS)
  1403. return;
  1404. /*
  1405. * mmap read-around
  1406. */
  1407. ra_pages = max_sane_readahead(ra->ra_pages);
  1408. ra->start = max_t(long, 0, offset - ra_pages / 2);
  1409. ra->size = ra_pages;
  1410. ra->async_size = ra_pages / 4;
  1411. ra_submit(ra, mapping, file);
  1412. }
  1413. /*
  1414. * Asynchronous readahead happens when we find the page and PG_readahead,
  1415. * so we want to possibly extend the readahead further..
  1416. */
  1417. static void do_async_mmap_readahead(struct vm_area_struct *vma,
  1418. struct file_ra_state *ra,
  1419. struct file *file,
  1420. struct page *page,
  1421. pgoff_t offset)
  1422. {
  1423. struct address_space *mapping = file->f_mapping;
  1424. /* If we don't want any read-ahead, don't bother */
  1425. if (vma->vm_flags & VM_RAND_READ)
  1426. return;
  1427. if (ra->mmap_miss > 0)
  1428. ra->mmap_miss--;
  1429. if (PageReadahead(page))
  1430. page_cache_async_readahead(mapping, ra, file,
  1431. page, offset, ra->ra_pages);
  1432. }
  1433. /**
  1434. * filemap_fault - read in file data for page fault handling
  1435. * @vma: vma in which the fault was taken
  1436. * @vmf: struct vm_fault containing details of the fault
  1437. *
  1438. * filemap_fault() is invoked via the vma operations vector for a
  1439. * mapped memory region to read in file data during a page fault.
  1440. *
  1441. * The goto's are kind of ugly, but this streamlines the normal case of having
  1442. * it in the page cache, and handles the special cases reasonably without
  1443. * having a lot of duplicated code.
  1444. */
  1445. int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  1446. {
  1447. int error;
  1448. struct file *file = vma->vm_file;
  1449. struct address_space *mapping = file->f_mapping;
  1450. struct file_ra_state *ra = &file->f_ra;
  1451. struct inode *inode = mapping->host;
  1452. pgoff_t offset = vmf->pgoff;
  1453. struct page *page;
  1454. pgoff_t size;
  1455. int ret = 0;
  1456. size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
  1457. if (offset >= size)
  1458. return VM_FAULT_SIGBUS;
  1459. /*
  1460. * Do we have something in the page cache already?
  1461. */
  1462. page = find_get_page(mapping, offset);
  1463. if (likely(page) && !(vmf->flags & FAULT_FLAG_TRIED)) {
  1464. /*
  1465. * We found the page, so try async readahead before
  1466. * waiting for the lock.
  1467. */
  1468. do_async_mmap_readahead(vma, ra, file, page, offset);
  1469. } else if (!page) {
  1470. /* No page in the page cache at all */
  1471. do_sync_mmap_readahead(vma, ra, file, offset);
  1472. count_vm_event(PGMAJFAULT);
  1473. mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
  1474. ret = VM_FAULT_MAJOR;
  1475. retry_find:
  1476. page = find_get_page(mapping, offset);
  1477. if (!page)
  1478. goto no_cached_page;
  1479. }
  1480. if (!lock_page_or_retry(page, vma->vm_mm, vmf->flags)) {
  1481. page_cache_release(page);
  1482. return ret | VM_FAULT_RETRY;
  1483. }
  1484. /* Did it get truncated? */
  1485. if (unlikely(page->mapping != mapping)) {
  1486. unlock_page(page);
  1487. put_page(page);
  1488. goto retry_find;
  1489. }
  1490. VM_BUG_ON(page->index != offset);
  1491. /*
  1492. * We have a locked page in the page cache, now we need to check
  1493. * that it's up-to-date. If not, it is going to be due to an error.
  1494. */
  1495. if (unlikely(!PageUptodate(page)))
  1496. goto page_not_uptodate;
  1497. /*
  1498. * Found the page and have a reference on it.
  1499. * We must recheck i_size under page lock.
  1500. */
  1501. size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
  1502. if (unlikely(offset >= size)) {
  1503. unlock_page(page);
  1504. page_cache_release(page);
  1505. return VM_FAULT_SIGBUS;
  1506. }
  1507. vmf->page = page;
  1508. return ret | VM_FAULT_LOCKED;
  1509. no_cached_page:
  1510. /*
  1511. * We're only likely to ever get here if MADV_RANDOM is in
  1512. * effect.
  1513. */
  1514. error = page_cache_read(file, offset);
  1515. /*
  1516. * The page we want has now been added to the page cache.
  1517. * In the unlikely event that someone removed it in the
  1518. * meantime, we'll just come back here and read it again.
  1519. */
  1520. if (error >= 0)
  1521. goto retry_find;
  1522. /*
  1523. * An error return from page_cache_read can result if the
  1524. * system is low on memory, or a problem occurs while trying
  1525. * to schedule I/O.
  1526. */
  1527. if (error == -ENOMEM)
  1528. return VM_FAULT_OOM;
  1529. return VM_FAULT_SIGBUS;
  1530. page_not_uptodate:
  1531. /*
  1532. * Umm, take care of errors if the page isn't up-to-date.
  1533. * Try to re-read it _once_. We do this synchronously,
  1534. * because there really aren't any performance issues here
  1535. * and we need to check for errors.
  1536. */
  1537. ClearPageError(page);
  1538. error = mapping->a_ops->readpage(file, page);
  1539. if (!error) {
  1540. wait_on_page_locked(page);
  1541. if (!PageUptodate(page))
  1542. error = -EIO;
  1543. }
  1544. page_cache_release(page);
  1545. if (!error || error == AOP_TRUNCATED_PAGE)
  1546. goto retry_find;
  1547. /* Things didn't work out. Return zero to tell the mm layer so. */
  1548. shrink_readahead_size_eio(file, ra);
  1549. return VM_FAULT_SIGBUS;
  1550. }
  1551. EXPORT_SYMBOL(filemap_fault);
  1552. int filemap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
  1553. {
  1554. struct page *page = vmf->page;
  1555. struct inode *inode = file_inode(vma->vm_file);
  1556. int ret = VM_FAULT_LOCKED;
  1557. sb_start_pagefault(inode->i_sb);
  1558. file_update_time(vma->vm_file);
  1559. lock_page(page);
  1560. if (page->mapping != inode->i_mapping) {
  1561. unlock_page(page);
  1562. ret = VM_FAULT_NOPAGE;
  1563. goto out;
  1564. }
  1565. /*
  1566. * We mark the page dirty already here so that when freeze is in
  1567. * progress, we are guaranteed that writeback during freezing will
  1568. * see the dirty page and writeprotect it again.
  1569. */
  1570. set_page_dirty(page);
  1571. wait_for_stable_page(page);
  1572. out:
  1573. sb_end_pagefault(inode->i_sb);
  1574. return ret;
  1575. }
  1576. EXPORT_SYMBOL(filemap_page_mkwrite);
  1577. const struct vm_operations_struct generic_file_vm_ops = {
  1578. .fault = filemap_fault,
  1579. .page_mkwrite = filemap_page_mkwrite,
  1580. .remap_pages = generic_file_remap_pages,
  1581. };
  1582. /* This is used for a general mmap of a disk file */
  1583. int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
  1584. {
  1585. struct address_space *mapping = file->f_mapping;
  1586. if (!mapping->a_ops->readpage)
  1587. return -ENOEXEC;
  1588. file_accessed(file);
  1589. vma->vm_ops = &generic_file_vm_ops;
  1590. return 0;
  1591. }
  1592. /*
  1593. * This is for filesystems which do not implement ->writepage.
  1594. */
  1595. int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma)
  1596. {
  1597. if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
  1598. return -EINVAL;
  1599. return generic_file_mmap(file, vma);
  1600. }
  1601. #else
  1602. int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
  1603. {
  1604. return -ENOSYS;
  1605. }
  1606. int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma)
  1607. {
  1608. return -ENOSYS;
  1609. }
  1610. #endif /* CONFIG_MMU */
  1611. EXPORT_SYMBOL(generic_file_mmap);
  1612. EXPORT_SYMBOL(generic_file_readonly_mmap);
  1613. static struct page *__read_cache_page(struct address_space *mapping,
  1614. pgoff_t index,
  1615. int (*filler)(void *, struct page *),
  1616. void *data,
  1617. gfp_t gfp)
  1618. {
  1619. struct page *page;
  1620. int err;
  1621. repeat:
  1622. page = find_get_page(mapping, index);
  1623. if (!page) {
  1624. page = __page_cache_alloc(gfp | __GFP_COLD);
  1625. if (!page)
  1626. return ERR_PTR(-ENOMEM);
  1627. err = add_to_page_cache_lru(page, mapping, index, gfp);
  1628. if (unlikely(err)) {
  1629. page_cache_release(page);
  1630. if (err == -EEXIST)
  1631. goto repeat;
  1632. /* Presumably ENOMEM for radix tree node */
  1633. return ERR_PTR(err);
  1634. }
  1635. err = filler(data, page);
  1636. if (err < 0) {
  1637. page_cache_release(page);
  1638. page = ERR_PTR(err);
  1639. }
  1640. }
  1641. return page;
  1642. }
  1643. static struct page *do_read_cache_page(struct address_space *mapping,
  1644. pgoff_t index,
  1645. int (*filler)(void *, struct page *),
  1646. void *data,
  1647. gfp_t gfp)
  1648. {
  1649. struct page *page;
  1650. int err;
  1651. retry:
  1652. page = __read_cache_page(mapping, index, filler, data, gfp);
  1653. if (IS_ERR(page))
  1654. return page;
  1655. if (PageUptodate(page))
  1656. goto out;
  1657. lock_page(page);
  1658. if (!page->mapping) {
  1659. unlock_page(page);
  1660. page_cache_release(page);
  1661. goto retry;
  1662. }
  1663. if (PageUptodate(page)) {
  1664. unlock_page(page);
  1665. goto out;
  1666. }
  1667. err = filler(data, page);
  1668. if (err < 0) {
  1669. page_cache_release(page);
  1670. return ERR_PTR(err);
  1671. }
  1672. out:
  1673. mark_page_accessed(page);
  1674. return page;
  1675. }
  1676. /**
  1677. * read_cache_page_async - read into page cache, fill it if needed
  1678. * @mapping: the page's address_space
  1679. * @index: the page index
  1680. * @filler: function to perform the read
  1681. * @data: first arg to filler(data, page) function, often left as NULL
  1682. *
  1683. * Same as read_cache_page, but don't wait for page to become unlocked
  1684. * after submitting it to the filler.
  1685. *
  1686. * Read into the page cache. If a page already exists, and PageUptodate() is
  1687. * not set, try to fill the page but don't wait for it to become unlocked.
  1688. *
  1689. * If the page does not get brought uptodate, return -EIO.
  1690. */
  1691. struct page *read_cache_page_async(struct address_space *mapping,
  1692. pgoff_t index,
  1693. int (*filler)(void *, struct page *),
  1694. void *data)
  1695. {
  1696. return do_read_cache_page(mapping, index, filler, data, mapping_gfp_mask(mapping));
  1697. }
  1698. EXPORT_SYMBOL(read_cache_page_async);
  1699. static struct page *wait_on_page_read(struct page *page)
  1700. {
  1701. if (!IS_ERR(page)) {
  1702. wait_on_page_locked(page);
  1703. if (!PageUptodate(page)) {
  1704. page_cache_release(page);
  1705. page = ERR_PTR(-EIO);
  1706. }
  1707. }
  1708. return page;
  1709. }
  1710. /**
  1711. * read_cache_page_gfp - read into page cache, using specified page allocation flags.
  1712. * @mapping: the page's address_space
  1713. * @index: the page index
  1714. * @gfp: the page allocator flags to use if allocating
  1715. *
  1716. * This is the same as "read_mapping_page(mapping, index, NULL)", but with
  1717. * any new page allocations done using the specified allocation flags.
  1718. *
  1719. * If the page does not get brought uptodate, return -EIO.
  1720. */
  1721. struct page *read_cache_page_gfp(struct address_space *mapping,
  1722. pgoff_t index,
  1723. gfp_t gfp)
  1724. {
  1725. filler_t *filler = (filler_t *)mapping->a_ops->readpage;
  1726. return wait_on_page_read(do_read_cache_page(mapping, index, filler, NULL, gfp));
  1727. }
  1728. EXPORT_SYMBOL(read_cache_page_gfp);
  1729. /**
  1730. * read_cache_page - read into page cache, fill it if needed
  1731. * @mapping: the page's address_space
  1732. * @index: the page index
  1733. * @filler: function to perform the read
  1734. * @data: first arg to filler(data, page) function, often left as NULL
  1735. *
  1736. * Read into the page cache. If a page already exists, and PageUptodate() is
  1737. * not set, try to fill the page then wait for it to become unlocked.
  1738. *
  1739. * If the page does not get brought uptodate, return -EIO.
  1740. */
  1741. struct page *read_cache_page(struct address_space *mapping,
  1742. pgoff_t index,
  1743. int (*filler)(void *, struct page *),
  1744. void *data)
  1745. {
  1746. return wait_on_page_read(read_cache_page_async(mapping, index, filler, data));
  1747. }
  1748. EXPORT_SYMBOL(read_cache_page);
  1749. static size_t __iovec_copy_from_user_inatomic(char *vaddr,
  1750. const struct iovec *iov, size_t base, size_t bytes)
  1751. {
  1752. size_t copied = 0, left = 0;
  1753. while (bytes) {
  1754. char __user *buf = iov->iov_base + base;
  1755. int copy = min(bytes, iov->iov_len - base);
  1756. base = 0;
  1757. left = __copy_from_user_inatomic(vaddr, buf, copy);
  1758. copied += copy;
  1759. bytes -= copy;
  1760. vaddr += copy;
  1761. iov++;
  1762. if (unlikely(left))
  1763. break;
  1764. }
  1765. return copied - left;
  1766. }
  1767. /*
  1768. * Copy as much as we can into the page and return the number of bytes which
  1769. * were successfully copied. If a fault is encountered then return the number of
  1770. * bytes which were copied.
  1771. */
  1772. size_t iov_iter_copy_from_user_atomic(struct page *page,
  1773. struct iov_iter *i, unsigned long offset, size_t bytes)
  1774. {
  1775. char *kaddr;
  1776. size_t copied;
  1777. BUG_ON(!in_atomic());
  1778. kaddr = kmap_atomic(page);
  1779. if (likely(i->nr_segs == 1)) {
  1780. int left;
  1781. char __user *buf = i->iov->iov_base + i->iov_offset;
  1782. left = __copy_from_user_inatomic(kaddr + offset, buf, bytes);
  1783. copied = bytes - left;
  1784. } else {
  1785. copied = __iovec_copy_from_user_inatomic(kaddr + offset,
  1786. i->iov, i->iov_offset, bytes);
  1787. }
  1788. kunmap_atomic(kaddr);
  1789. return copied;
  1790. }
  1791. EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
  1792. /*
  1793. * This has the same sideeffects and return value as
  1794. * iov_iter_copy_from_user_atomic().
  1795. * The difference is that it attempts to resolve faults.
  1796. * Page must not be locked.
  1797. */
  1798. size_t iov_iter_copy_from_user(struct page *page,
  1799. struct iov_iter *i, unsigned long offset, size_t bytes)
  1800. {
  1801. char *kaddr;
  1802. size_t copied;
  1803. kaddr = kmap(page);
  1804. if (likely(i->nr_segs == 1)) {
  1805. int left;
  1806. char __user *buf = i->iov->iov_base + i->iov_offset;
  1807. left = __copy_from_user(kaddr + offset, buf, bytes);
  1808. copied = bytes - left;
  1809. } else {
  1810. copied = __iovec_copy_from_user_inatomic(kaddr + offset,
  1811. i->iov, i->iov_offset, bytes);
  1812. }
  1813. kunmap(page);
  1814. return copied;
  1815. }
  1816. EXPORT_SYMBOL(iov_iter_copy_from_user);
  1817. void iov_iter_advance(struct iov_iter *i, size_t bytes)
  1818. {
  1819. BUG_ON(i->count < bytes);
  1820. if (likely(i->nr_segs == 1)) {
  1821. i->iov_offset += bytes;
  1822. i->count -= bytes;
  1823. } else {
  1824. const struct iovec *iov = i->iov;
  1825. size_t base = i->iov_offset;
  1826. unsigned long nr_segs = i->nr_segs;
  1827. /*
  1828. * The !iov->iov_len check ensures we skip over unlikely
  1829. * zero-length segments (without overruning the iovec).
  1830. */
  1831. while (bytes || unlikely(i->count && !iov->iov_len)) {
  1832. int copy;
  1833. copy = min(bytes, iov->iov_len - base);
  1834. BUG_ON(!i->count || i->count < copy);
  1835. i->count -= copy;
  1836. bytes -= copy;
  1837. base += copy;
  1838. if (iov->iov_len == base) {
  1839. iov++;
  1840. nr_segs--;
  1841. base = 0;
  1842. }
  1843. }
  1844. i->iov = iov;
  1845. i->iov_offset = base;
  1846. i->nr_segs = nr_segs;
  1847. }
  1848. }
  1849. EXPORT_SYMBOL(iov_iter_advance);
  1850. /*
  1851. * Fault in the first iovec of the given iov_iter, to a maximum length
  1852. * of bytes. Returns 0 on success, or non-zero if the memory could not be
  1853. * accessed (ie. because it is an invalid address).
  1854. *
  1855. * writev-intensive code may want this to prefault several iovecs -- that
  1856. * would be possible (callers must not rely on the fact that _only_ the
  1857. * first iovec will be faulted with the current implementation).
  1858. */
  1859. int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
  1860. {
  1861. char __user *buf = i->iov->iov_base + i->iov_offset;
  1862. bytes = min(bytes, i->iov->iov_len - i->iov_offset);
  1863. return fault_in_pages_readable(buf, bytes);
  1864. }
  1865. EXPORT_SYMBOL(iov_iter_fault_in_readable);
  1866. /*
  1867. * Return the count of just the current iov_iter segment.
  1868. */
  1869. size_t iov_iter_single_seg_count(const struct iov_iter *i)
  1870. {
  1871. const struct iovec *iov = i->iov;
  1872. if (i->nr_segs == 1)
  1873. return i->count;
  1874. else
  1875. return min(i->count, iov->iov_len - i->iov_offset);
  1876. }
  1877. EXPORT_SYMBOL(iov_iter_single_seg_count);
  1878. /*
  1879. * Performs necessary checks before doing a write
  1880. *
  1881. * Can adjust writing position or amount of bytes to write.
  1882. * Returns appropriate error code that caller should return or
  1883. * zero in case that write should be allowed.
  1884. */
  1885. inline int generic_write_checks(struct file *file, loff_t *pos, size_t *count, int isblk)
  1886. {
  1887. struct inode *inode = file->f_mapping->host;
  1888. unsigned long limit = rlimit(RLIMIT_FSIZE);
  1889. if (unlikely(*pos < 0))
  1890. return -EINVAL;
  1891. if (!isblk) {
  1892. /* FIXME: this is for backwards compatibility with 2.4 */
  1893. if (file->f_flags & O_APPEND)
  1894. *pos = i_size_read(inode);
  1895. if (limit != RLIM_INFINITY) {
  1896. if (*pos >= limit) {
  1897. send_sig(SIGXFSZ, current, 0);
  1898. return -EFBIG;
  1899. }
  1900. if (*count > limit - (typeof(limit))*pos) {
  1901. *count = limit - (typeof(limit))*pos;
  1902. }
  1903. }
  1904. }
  1905. /*
  1906. * LFS rule
  1907. */
  1908. if (unlikely(*pos + *count > MAX_NON_LFS &&
  1909. !(file->f_flags & O_LARGEFILE))) {
  1910. if (*pos >= MAX_NON_LFS) {
  1911. return -EFBIG;
  1912. }
  1913. if (*count > MAX_NON_LFS - (unsigned long)*pos) {
  1914. *count = MAX_NON_LFS - (unsigned long)*pos;
  1915. }
  1916. }
  1917. /*
  1918. * Are we about to exceed the fs block limit ?
  1919. *
  1920. * If we have written data it becomes a short write. If we have
  1921. * exceeded without writing data we send a signal and return EFBIG.
  1922. * Linus frestrict idea will clean these up nicely..
  1923. */
  1924. if (likely(!isblk)) {
  1925. if (unlikely(*pos >= inode->i_sb->s_maxbytes)) {
  1926. if (*count || *pos > inode->i_sb->s_maxbytes) {
  1927. return -EFBIG;
  1928. }
  1929. /* zero-length writes at ->s_maxbytes are OK */
  1930. }
  1931. if (unlikely(*pos + *count > inode->i_sb->s_maxbytes))
  1932. *count = inode->i_sb->s_maxbytes - *pos;
  1933. } else {
  1934. #ifdef CONFIG_BLOCK
  1935. loff_t isize;
  1936. if (bdev_read_only(I_BDEV(inode)))
  1937. return -EPERM;
  1938. isize = i_size_read(inode);
  1939. if (*pos >= isize) {
  1940. if (*count || *pos > isize)
  1941. return -ENOSPC;
  1942. }
  1943. if (*pos + *count > isize)
  1944. *count = isize - *pos;
  1945. #else
  1946. return -EPERM;
  1947. #endif
  1948. }
  1949. return 0;
  1950. }
  1951. EXPORT_SYMBOL(generic_write_checks);
  1952. int pagecache_write_begin(struct file *file, struct address_space *mapping,
  1953. loff_t pos, unsigned len, unsigned flags,
  1954. struct page **pagep, void **fsdata)
  1955. {
  1956. const struct address_space_operations *aops = mapping->a_ops;
  1957. return aops->write_begin(file, mapping, pos, len, flags,
  1958. pagep, fsdata);
  1959. }
  1960. EXPORT_SYMBOL(pagecache_write_begin);
  1961. int pagecache_write_end(struct file *file, struct address_space *mapping,
  1962. loff_t pos, unsigned len, unsigned copied,
  1963. struct page *page, void *fsdata)
  1964. {
  1965. const struct address_space_operations *aops = mapping->a_ops;
  1966. mark_page_accessed(page);
  1967. return aops->write_end(file, mapping, pos, len, copied, page, fsdata);
  1968. }
  1969. EXPORT_SYMBOL(pagecache_write_end);
  1970. ssize_t
  1971. generic_file_direct_write(struct kiocb *iocb, const struct iovec *iov,
  1972. unsigned long *nr_segs, loff_t pos, loff_t *ppos,
  1973. size_t count, size_t ocount)
  1974. {
  1975. struct file *file = iocb->ki_filp;
  1976. struct address_space *mapping = file->f_mapping;
  1977. struct inode *inode = mapping->host;
  1978. ssize_t written;
  1979. size_t write_len;
  1980. pgoff_t end;
  1981. if (count != ocount)
  1982. *nr_segs = iov_shorten((struct iovec *)iov, *nr_segs, count);
  1983. write_len = iov_length(iov, *nr_segs);
  1984. end = (pos + write_len - 1) >> PAGE_CACHE_SHIFT;
  1985. written = filemap_write_and_wait_range(mapping, pos, pos + write_len - 1);
  1986. if (written)
  1987. goto out;
  1988. /*
  1989. * After a write we want buffered reads to be sure to go to disk to get
  1990. * the new data. We invalidate clean cached page from the region we're
  1991. * about to write. We do this *before* the write so that we can return
  1992. * without clobbering -EIOCBQUEUED from ->direct_IO().
  1993. */
  1994. if (mapping->nrpages) {
  1995. written = invalidate_inode_pages2_range(mapping,
  1996. pos >> PAGE_CACHE_SHIFT, end);
  1997. /*
  1998. * If a page can not be invalidated, return 0 to fall back
  1999. * to buffered write.
  2000. */
  2001. if (written) {
  2002. if (written == -EBUSY)
  2003. return 0;
  2004. goto out;
  2005. }
  2006. }
  2007. written = mapping->a_ops->direct_IO(WRITE, iocb, iov, pos, *nr_segs);
  2008. /*
  2009. * Finally, try again to invalidate clean pages which might have been
  2010. * cached by non-direct readahead, or faulted in by get_user_pages()
  2011. * if the source of the write was an mmap'ed region of the file
  2012. * we're writing. Either one is a pretty crazy thing to do,
  2013. * so we don't support it 100%. If this invalidation
  2014. * fails, tough, the write still worked...
  2015. */
  2016. if (mapping->nrpages) {
  2017. invalidate_inode_pages2_range(mapping,
  2018. pos >> PAGE_CACHE_SHIFT, end);
  2019. }
  2020. if (written > 0) {
  2021. pos += written;
  2022. if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) {
  2023. i_size_write(inode, pos);
  2024. mark_inode_dirty(inode);
  2025. }
  2026. *ppos = pos;
  2027. }
  2028. out:
  2029. return written;
  2030. }
  2031. EXPORT_SYMBOL(generic_file_direct_write);
  2032. /*
  2033. * Find or create a page at the given pagecache position. Return the locked
  2034. * page. This function is specifically for buffered writes.
  2035. */
  2036. struct page *grab_cache_page_write_begin(struct address_space *mapping,
  2037. pgoff_t index, unsigned flags)
  2038. {
  2039. int status;
  2040. gfp_t gfp_mask;
  2041. struct page *page;
  2042. gfp_t gfp_notmask = 0;
  2043. gfp_mask = mapping_gfp_mask(mapping);
  2044. if (mapping_cap_account_dirty(mapping))
  2045. gfp_mask |= __GFP_WRITE;
  2046. if (flags & AOP_FLAG_NOFS)
  2047. gfp_notmask = __GFP_FS;
  2048. repeat:
  2049. page = find_lock_page(mapping, index);
  2050. if (page)
  2051. goto found;
  2052. page = __page_cache_alloc(gfp_mask & ~gfp_notmask);
  2053. if (!page)
  2054. return NULL;
  2055. status = add_to_page_cache_lru(page, mapping, index,
  2056. GFP_KERNEL & ~gfp_notmask);
  2057. if (unlikely(status)) {
  2058. page_cache_release(page);
  2059. if (status == -EEXIST)
  2060. goto repeat;
  2061. return NULL;
  2062. }
  2063. found:
  2064. wait_for_stable_page(page);
  2065. return page;
  2066. }
  2067. EXPORT_SYMBOL(grab_cache_page_write_begin);
  2068. static ssize_t generic_perform_write(struct file *file,
  2069. struct iov_iter *i, loff_t pos)
  2070. {
  2071. struct address_space *mapping = file->f_mapping;
  2072. const struct address_space_operations *a_ops = mapping->a_ops;
  2073. long status = 0;
  2074. ssize_t written = 0;
  2075. unsigned int flags = 0;
  2076. /*
  2077. * Copies from kernel address space cannot fail (NFSD is a big user).
  2078. */
  2079. if (segment_eq(get_fs(), KERNEL_DS))
  2080. flags |= AOP_FLAG_UNINTERRUPTIBLE;
  2081. do {
  2082. struct page *page;
  2083. unsigned long offset; /* Offset into pagecache page */
  2084. unsigned long bytes; /* Bytes to write to page */
  2085. size_t copied; /* Bytes copied from user */
  2086. void *fsdata;
  2087. offset = (pos & (PAGE_CACHE_SIZE - 1));
  2088. bytes = min_t(unsigned long, PAGE_CACHE_SIZE - offset,
  2089. iov_iter_count(i));
  2090. again:
  2091. /*
  2092. * Bring in the user page that we will copy from _first_.
  2093. * Otherwise there's a nasty deadlock on copying from the
  2094. * same page as we're writing to, without it being marked
  2095. * up-to-date.
  2096. *
  2097. * Not only is this an optimisation, but it is also required
  2098. * to check that the address is actually valid, when atomic
  2099. * usercopies are used, below.
  2100. */
  2101. if (unlikely(iov_iter_fault_in_readable(i, bytes))) {
  2102. status = -EFAULT;
  2103. break;
  2104. }
  2105. status = a_ops->write_begin(file, mapping, pos, bytes, flags,
  2106. &page, &fsdata);
  2107. if (unlikely(status))
  2108. break;
  2109. if (mapping_writably_mapped(mapping))
  2110. flush_dcache_page(page);
  2111. pagefault_disable();
  2112. copied = iov_iter_copy_from_user_atomic(page, i, offset, bytes);
  2113. pagefault_enable();
  2114. flush_dcache_page(page);
  2115. mark_page_accessed(page);
  2116. status = a_ops->write_end(file, mapping, pos, bytes, copied,
  2117. page, fsdata);
  2118. if (unlikely(status < 0))
  2119. break;
  2120. copied = status;
  2121. cond_resched();
  2122. iov_iter_advance(i, copied);
  2123. if (unlikely(copied == 0)) {
  2124. /*
  2125. * If we were unable to copy any data at all, we must
  2126. * fall back to a single segment length write.
  2127. *
  2128. * If we didn't fallback here, we could livelock
  2129. * because not all segments in the iov can be copied at
  2130. * once without a pagefault.
  2131. */
  2132. bytes = min_t(unsigned long, PAGE_CACHE_SIZE - offset,
  2133. iov_iter_single_seg_count(i));
  2134. goto again;
  2135. }
  2136. pos += copied;
  2137. written += copied;
  2138. balance_dirty_pages_ratelimited(mapping);
  2139. if (fatal_signal_pending(current)) {
  2140. status = -EINTR;
  2141. break;
  2142. }
  2143. } while (iov_iter_count(i));
  2144. return written ? written : status;
  2145. }
  2146. ssize_t
  2147. generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
  2148. unsigned long nr_segs, loff_t pos, loff_t *ppos,
  2149. size_t count, ssize_t written)
  2150. {
  2151. struct file *file = iocb->ki_filp;
  2152. ssize_t status;
  2153. struct iov_iter i;
  2154. iov_iter_init(&i, iov, nr_segs, count, written);
  2155. status = generic_perform_write(file, &i, pos);
  2156. if (likely(status >= 0)) {
  2157. written += status;
  2158. *ppos = pos + status;
  2159. }
  2160. return written ? written : status;
  2161. }
  2162. EXPORT_SYMBOL(generic_file_buffered_write);
  2163. /**
  2164. * __generic_file_aio_write - write data to a file
  2165. * @iocb: IO state structure (file, offset, etc.)
  2166. * @iov: vector with data to write
  2167. * @nr_segs: number of segments in the vector
  2168. * @ppos: position where to write
  2169. *
  2170. * This function does all the work needed for actually writing data to a
  2171. * file. It does all basic checks, removes SUID from the file, updates
  2172. * modification times and calls proper subroutines depending on whether we
  2173. * do direct IO or a standard buffered write.
  2174. *
  2175. * It expects i_mutex to be grabbed unless we work on a block device or similar
  2176. * object which does not need locking at all.
  2177. *
  2178. * This function does *not* take care of syncing data in case of O_SYNC write.
  2179. * A caller has to handle it. This is mainly due to the fact that we want to
  2180. * avoid syncing under i_mutex.
  2181. */
  2182. ssize_t __generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
  2183. unsigned long nr_segs, loff_t *ppos)
  2184. {
  2185. struct file *file = iocb->ki_filp;
  2186. struct address_space * mapping = file->f_mapping;
  2187. size_t ocount; /* original count */
  2188. size_t count; /* after file limit checks */
  2189. struct inode *inode = mapping->host;
  2190. loff_t pos;
  2191. ssize_t written;
  2192. ssize_t err;
  2193. ocount = 0;
  2194. err = generic_segment_checks(iov, &nr_segs, &ocount, VERIFY_READ);
  2195. if (err)
  2196. return err;
  2197. count = ocount;
  2198. pos = *ppos;
  2199. /* We can write back this queue in page reclaim */
  2200. current->backing_dev_info = mapping->backing_dev_info;
  2201. written = 0;
  2202. err = generic_write_checks(file, &pos, &count, S_ISBLK(inode->i_mode));
  2203. if (err)
  2204. goto out;
  2205. if (count == 0)
  2206. goto out;
  2207. err = file_remove_suid(file);
  2208. if (err)
  2209. goto out;
  2210. err = file_update_time(file);
  2211. if (err)
  2212. goto out;
  2213. /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
  2214. if (unlikely(file->f_flags & O_DIRECT)) {
  2215. loff_t endbyte;
  2216. ssize_t written_buffered;
  2217. written = generic_file_direct_write(iocb, iov, &nr_segs, pos,
  2218. ppos, count, ocount);
  2219. if (written < 0 || written == count)
  2220. goto out;
  2221. /*
  2222. * direct-io write to a hole: fall through to buffered I/O
  2223. * for completing the rest of the request.
  2224. */
  2225. pos += written;
  2226. count -= written;
  2227. written_buffered = generic_file_buffered_write(iocb, iov,
  2228. nr_segs, pos, ppos, count,
  2229. written);
  2230. /*
  2231. * If generic_file_buffered_write() retuned a synchronous error
  2232. * then we want to return the number of bytes which were
  2233. * direct-written, or the error code if that was zero. Note
  2234. * that this differs from normal direct-io semantics, which
  2235. * will return -EFOO even if some bytes were written.
  2236. */
  2237. if (written_buffered < 0) {
  2238. err = written_buffered;
  2239. goto out;
  2240. }
  2241. /*
  2242. * We need to ensure that the page cache pages are written to
  2243. * disk and invalidated to preserve the expected O_DIRECT
  2244. * semantics.
  2245. */
  2246. endbyte = pos + written_buffered - written - 1;
  2247. err = filemap_write_and_wait_range(file->f_mapping, pos, endbyte);
  2248. if (err == 0) {
  2249. written = written_buffered;
  2250. invalidate_mapping_pages(mapping,
  2251. pos >> PAGE_CACHE_SHIFT,
  2252. endbyte >> PAGE_CACHE_SHIFT);
  2253. } else {
  2254. /*
  2255. * We don't know how much we wrote, so just return
  2256. * the number of bytes which were direct-written
  2257. */
  2258. }
  2259. } else {
  2260. written = generic_file_buffered_write(iocb, iov, nr_segs,
  2261. pos, ppos, count, written);
  2262. }
  2263. out:
  2264. current->backing_dev_info = NULL;
  2265. return written ? written : err;
  2266. }
  2267. EXPORT_SYMBOL(__generic_file_aio_write);
  2268. /**
  2269. * generic_file_aio_write - write data to a file
  2270. * @iocb: IO state structure
  2271. * @iov: vector with data to write
  2272. * @nr_segs: number of segments in the vector
  2273. * @pos: position in file where to write
  2274. *
  2275. * This is a wrapper around __generic_file_aio_write() to be used by most
  2276. * filesystems. It takes care of syncing the file in case of O_SYNC file
  2277. * and acquires i_mutex as needed.
  2278. */
  2279. ssize_t generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
  2280. unsigned long nr_segs, loff_t pos)
  2281. {
  2282. struct file *file = iocb->ki_filp;
  2283. struct inode *inode = file->f_mapping->host;
  2284. ssize_t ret;
  2285. BUG_ON(iocb->ki_pos != pos);
  2286. mutex_lock(&inode->i_mutex);
  2287. ret = __generic_file_aio_write(iocb, iov, nr_segs, &iocb->ki_pos);
  2288. mutex_unlock(&inode->i_mutex);
  2289. if (ret > 0) {
  2290. ssize_t err;
  2291. err = generic_write_sync(file, pos, ret);
  2292. if (err < 0 && ret > 0)
  2293. ret = err;
  2294. }
  2295. return ret;
  2296. }
  2297. EXPORT_SYMBOL(generic_file_aio_write);
  2298. /**
  2299. * try_to_release_page() - release old fs-specific metadata on a page
  2300. *
  2301. * @page: the page which the kernel is trying to free
  2302. * @gfp_mask: memory allocation flags (and I/O mode)
  2303. *
  2304. * The address_space is to try to release any data against the page
  2305. * (presumably at page->private). If the release was successful, return `1'.
  2306. * Otherwise return zero.
  2307. *
  2308. * This may also be called if PG_fscache is set on a page, indicating that the
  2309. * page is known to the local caching routines.
  2310. *
  2311. * The @gfp_mask argument specifies whether I/O may be performed to release
  2312. * this page (__GFP_IO), and whether the call may block (__GFP_WAIT & __GFP_FS).
  2313. *
  2314. */
  2315. int try_to_release_page(struct page *page, gfp_t gfp_mask)
  2316. {
  2317. struct address_space * const mapping = page->mapping;
  2318. BUG_ON(!PageLocked(page));
  2319. if (PageWriteback(page))
  2320. return 0;
  2321. if (mapping && mapping->a_ops->releasepage)
  2322. return mapping->a_ops->releasepage(page, gfp_mask);
  2323. return try_to_free_buffers(page);
  2324. }
  2325. EXPORT_SYMBOL(try_to_release_page);