buffer.c 89 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438
  1. /*
  2. * linux/fs/buffer.c
  3. *
  4. * Copyright (C) 1991, 1992, 2002 Linus Torvalds
  5. */
  6. /*
  7. * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
  8. *
  9. * Removed a lot of unnecessary code and simplified things now that
  10. * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
  11. *
  12. * Speed up hash, lru, and free list operations. Use gfp() for allocating
  13. * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
  14. *
  15. * Added 32k buffer block sizes - these are required older ARM systems. - RMK
  16. *
  17. * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
  18. */
  19. #include <linux/kernel.h>
  20. #include <linux/syscalls.h>
  21. #include <linux/fs.h>
  22. #include <linux/mm.h>
  23. #include <linux/percpu.h>
  24. #include <linux/slab.h>
  25. #include <linux/capability.h>
  26. #include <linux/blkdev.h>
  27. #include <linux/file.h>
  28. #include <linux/quotaops.h>
  29. #include <linux/highmem.h>
  30. #include <linux/export.h>
  31. #include <linux/writeback.h>
  32. #include <linux/hash.h>
  33. #include <linux/suspend.h>
  34. #include <linux/buffer_head.h>
  35. #include <linux/task_io_accounting_ops.h>
  36. #include <linux/bio.h>
  37. #include <linux/notifier.h>
  38. #include <linux/cpu.h>
  39. #include <linux/bitops.h>
  40. #include <linux/mpage.h>
  41. #include <linux/bit_spinlock.h>
  42. #include <trace/events/block.h>
  43. static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
  44. #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
  45. void init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
  46. {
  47. bh->b_end_io = handler;
  48. bh->b_private = private;
  49. }
  50. EXPORT_SYMBOL(init_buffer);
  51. inline void touch_buffer(struct buffer_head *bh)
  52. {
  53. trace_block_touch_buffer(bh);
  54. mark_page_accessed(bh->b_page);
  55. }
  56. EXPORT_SYMBOL(touch_buffer);
  57. static int sleep_on_buffer(void *word)
  58. {
  59. io_schedule();
  60. return 0;
  61. }
  62. void __lock_buffer(struct buffer_head *bh)
  63. {
  64. wait_on_bit_lock(&bh->b_state, BH_Lock, sleep_on_buffer,
  65. TASK_UNINTERRUPTIBLE);
  66. }
  67. EXPORT_SYMBOL(__lock_buffer);
  68. void unlock_buffer(struct buffer_head *bh)
  69. {
  70. clear_bit_unlock(BH_Lock, &bh->b_state);
  71. smp_mb__after_clear_bit();
  72. wake_up_bit(&bh->b_state, BH_Lock);
  73. }
  74. EXPORT_SYMBOL(unlock_buffer);
  75. /*
  76. * Returns if the page has dirty or writeback buffers. If all the buffers
  77. * are unlocked and clean then the PageDirty information is stale. If
  78. * any of the pages are locked, it is assumed they are locked for IO.
  79. */
  80. void buffer_check_dirty_writeback(struct page *page,
  81. bool *dirty, bool *writeback)
  82. {
  83. struct buffer_head *head, *bh;
  84. *dirty = false;
  85. *writeback = false;
  86. BUG_ON(!PageLocked(page));
  87. if (!page_has_buffers(page))
  88. return;
  89. if (PageWriteback(page))
  90. *writeback = true;
  91. head = page_buffers(page);
  92. bh = head;
  93. do {
  94. if (buffer_locked(bh))
  95. *writeback = true;
  96. if (buffer_dirty(bh))
  97. *dirty = true;
  98. bh = bh->b_this_page;
  99. } while (bh != head);
  100. }
  101. EXPORT_SYMBOL(buffer_check_dirty_writeback);
  102. /*
  103. * Block until a buffer comes unlocked. This doesn't stop it
  104. * from becoming locked again - you have to lock it yourself
  105. * if you want to preserve its state.
  106. */
  107. void __wait_on_buffer(struct buffer_head * bh)
  108. {
  109. wait_on_bit(&bh->b_state, BH_Lock, sleep_on_buffer, TASK_UNINTERRUPTIBLE);
  110. }
  111. EXPORT_SYMBOL(__wait_on_buffer);
  112. static void
  113. __clear_page_buffers(struct page *page)
  114. {
  115. ClearPagePrivate(page);
  116. set_page_private(page, 0);
  117. page_cache_release(page);
  118. }
  119. static int quiet_error(struct buffer_head *bh)
  120. {
  121. if (!test_bit(BH_Quiet, &bh->b_state) && printk_ratelimit())
  122. return 0;
  123. return 1;
  124. }
  125. static void buffer_io_error(struct buffer_head *bh)
  126. {
  127. char b[BDEVNAME_SIZE];
  128. printk(KERN_ERR "Buffer I/O error on device %s, logical block %Lu\n",
  129. bdevname(bh->b_bdev, b),
  130. (unsigned long long)bh->b_blocknr);
  131. }
  132. /*
  133. * End-of-IO handler helper function which does not touch the bh after
  134. * unlocking it.
  135. * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
  136. * a race there is benign: unlock_buffer() only use the bh's address for
  137. * hashing after unlocking the buffer, so it doesn't actually touch the bh
  138. * itself.
  139. */
  140. static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
  141. {
  142. if (uptodate) {
  143. set_buffer_uptodate(bh);
  144. } else {
  145. /* This happens, due to failed READA attempts. */
  146. clear_buffer_uptodate(bh);
  147. }
  148. unlock_buffer(bh);
  149. }
  150. /*
  151. * Default synchronous end-of-IO handler.. Just mark it up-to-date and
  152. * unlock the buffer. This is what ll_rw_block uses too.
  153. */
  154. void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
  155. {
  156. __end_buffer_read_notouch(bh, uptodate);
  157. put_bh(bh);
  158. }
  159. EXPORT_SYMBOL(end_buffer_read_sync);
  160. void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
  161. {
  162. char b[BDEVNAME_SIZE];
  163. if (uptodate) {
  164. set_buffer_uptodate(bh);
  165. } else {
  166. if (!quiet_error(bh)) {
  167. buffer_io_error(bh);
  168. printk(KERN_WARNING "lost page write due to "
  169. "I/O error on %s\n",
  170. bdevname(bh->b_bdev, b));
  171. }
  172. set_buffer_write_io_error(bh);
  173. clear_buffer_uptodate(bh);
  174. }
  175. unlock_buffer(bh);
  176. put_bh(bh);
  177. }
  178. EXPORT_SYMBOL(end_buffer_write_sync);
  179. /*
  180. * Various filesystems appear to want __find_get_block to be non-blocking.
  181. * But it's the page lock which protects the buffers. To get around this,
  182. * we get exclusion from try_to_free_buffers with the blockdev mapping's
  183. * private_lock.
  184. *
  185. * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
  186. * may be quite high. This code could TryLock the page, and if that
  187. * succeeds, there is no need to take private_lock. (But if
  188. * private_lock is contended then so is mapping->tree_lock).
  189. */
  190. static struct buffer_head *
  191. __find_get_block_slow(struct block_device *bdev, sector_t block)
  192. {
  193. struct inode *bd_inode = bdev->bd_inode;
  194. struct address_space *bd_mapping = bd_inode->i_mapping;
  195. struct buffer_head *ret = NULL;
  196. pgoff_t index;
  197. struct buffer_head *bh;
  198. struct buffer_head *head;
  199. struct page *page;
  200. int all_mapped = 1;
  201. index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits);
  202. page = find_get_page(bd_mapping, index);
  203. if (!page)
  204. goto out;
  205. spin_lock(&bd_mapping->private_lock);
  206. if (!page_has_buffers(page))
  207. goto out_unlock;
  208. head = page_buffers(page);
  209. bh = head;
  210. do {
  211. if (!buffer_mapped(bh))
  212. all_mapped = 0;
  213. else if (bh->b_blocknr == block) {
  214. ret = bh;
  215. get_bh(bh);
  216. goto out_unlock;
  217. }
  218. bh = bh->b_this_page;
  219. } while (bh != head);
  220. /* we might be here because some of the buffers on this page are
  221. * not mapped. This is due to various races between
  222. * file io on the block device and getblk. It gets dealt with
  223. * elsewhere, don't buffer_error if we had some unmapped buffers
  224. */
  225. if (all_mapped) {
  226. char b[BDEVNAME_SIZE];
  227. printk("__find_get_block_slow() failed. "
  228. "block=%llu, b_blocknr=%llu\n",
  229. (unsigned long long)block,
  230. (unsigned long long)bh->b_blocknr);
  231. printk("b_state=0x%08lx, b_size=%zu\n",
  232. bh->b_state, bh->b_size);
  233. printk("device %s blocksize: %d\n", bdevname(bdev, b),
  234. 1 << bd_inode->i_blkbits);
  235. }
  236. out_unlock:
  237. spin_unlock(&bd_mapping->private_lock);
  238. page_cache_release(page);
  239. out:
  240. return ret;
  241. }
  242. /*
  243. * Kick the writeback threads then try to free up some ZONE_NORMAL memory.
  244. */
  245. static void free_more_memory(void)
  246. {
  247. struct zone *zone;
  248. int nid;
  249. wakeup_flusher_threads(1024, WB_REASON_FREE_MORE_MEM);
  250. yield();
  251. for_each_online_node(nid) {
  252. (void)first_zones_zonelist(node_zonelist(nid, GFP_NOFS),
  253. gfp_zone(GFP_NOFS), NULL,
  254. &zone);
  255. if (zone)
  256. try_to_free_pages(node_zonelist(nid, GFP_NOFS), 0,
  257. GFP_NOFS, NULL);
  258. }
  259. }
  260. /*
  261. * I/O completion handler for block_read_full_page() - pages
  262. * which come unlocked at the end of I/O.
  263. */
  264. static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
  265. {
  266. unsigned long flags;
  267. struct buffer_head *first;
  268. struct buffer_head *tmp;
  269. struct page *page;
  270. int page_uptodate = 1;
  271. BUG_ON(!buffer_async_read(bh));
  272. page = bh->b_page;
  273. if (uptodate) {
  274. set_buffer_uptodate(bh);
  275. } else {
  276. clear_buffer_uptodate(bh);
  277. if (!quiet_error(bh))
  278. buffer_io_error(bh);
  279. SetPageError(page);
  280. }
  281. /*
  282. * Be _very_ careful from here on. Bad things can happen if
  283. * two buffer heads end IO at almost the same time and both
  284. * decide that the page is now completely done.
  285. */
  286. first = page_buffers(page);
  287. local_irq_save(flags);
  288. bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
  289. clear_buffer_async_read(bh);
  290. unlock_buffer(bh);
  291. tmp = bh;
  292. do {
  293. if (!buffer_uptodate(tmp))
  294. page_uptodate = 0;
  295. if (buffer_async_read(tmp)) {
  296. BUG_ON(!buffer_locked(tmp));
  297. goto still_busy;
  298. }
  299. tmp = tmp->b_this_page;
  300. } while (tmp != bh);
  301. bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
  302. local_irq_restore(flags);
  303. /*
  304. * If none of the buffers had errors and they are all
  305. * uptodate then we can set the page uptodate.
  306. */
  307. if (page_uptodate && !PageError(page))
  308. SetPageUptodate(page);
  309. unlock_page(page);
  310. return;
  311. still_busy:
  312. bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
  313. local_irq_restore(flags);
  314. return;
  315. }
  316. /*
  317. * Completion handler for block_write_full_page() - pages which are unlocked
  318. * during I/O, and which have PageWriteback cleared upon I/O completion.
  319. */
  320. void end_buffer_async_write(struct buffer_head *bh, int uptodate)
  321. {
  322. char b[BDEVNAME_SIZE];
  323. unsigned long flags;
  324. struct buffer_head *first;
  325. struct buffer_head *tmp;
  326. struct page *page;
  327. BUG_ON(!buffer_async_write(bh));
  328. page = bh->b_page;
  329. if (uptodate) {
  330. set_buffer_uptodate(bh);
  331. } else {
  332. if (!quiet_error(bh)) {
  333. buffer_io_error(bh);
  334. printk(KERN_WARNING "lost page write due to "
  335. "I/O error on %s\n",
  336. bdevname(bh->b_bdev, b));
  337. }
  338. set_bit(AS_EIO, &page->mapping->flags);
  339. set_buffer_write_io_error(bh);
  340. clear_buffer_uptodate(bh);
  341. SetPageError(page);
  342. }
  343. first = page_buffers(page);
  344. local_irq_save(flags);
  345. bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
  346. clear_buffer_async_write(bh);
  347. unlock_buffer(bh);
  348. tmp = bh->b_this_page;
  349. while (tmp != bh) {
  350. if (buffer_async_write(tmp)) {
  351. BUG_ON(!buffer_locked(tmp));
  352. goto still_busy;
  353. }
  354. tmp = tmp->b_this_page;
  355. }
  356. bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
  357. local_irq_restore(flags);
  358. end_page_writeback(page);
  359. return;
  360. still_busy:
  361. bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
  362. local_irq_restore(flags);
  363. return;
  364. }
  365. EXPORT_SYMBOL(end_buffer_async_write);
  366. /*
  367. * If a page's buffers are under async readin (end_buffer_async_read
  368. * completion) then there is a possibility that another thread of
  369. * control could lock one of the buffers after it has completed
  370. * but while some of the other buffers have not completed. This
  371. * locked buffer would confuse end_buffer_async_read() into not unlocking
  372. * the page. So the absence of BH_Async_Read tells end_buffer_async_read()
  373. * that this buffer is not under async I/O.
  374. *
  375. * The page comes unlocked when it has no locked buffer_async buffers
  376. * left.
  377. *
  378. * PageLocked prevents anyone starting new async I/O reads any of
  379. * the buffers.
  380. *
  381. * PageWriteback is used to prevent simultaneous writeout of the same
  382. * page.
  383. *
  384. * PageLocked prevents anyone from starting writeback of a page which is
  385. * under read I/O (PageWriteback is only ever set against a locked page).
  386. */
  387. static void mark_buffer_async_read(struct buffer_head *bh)
  388. {
  389. bh->b_end_io = end_buffer_async_read;
  390. set_buffer_async_read(bh);
  391. }
  392. static void mark_buffer_async_write_endio(struct buffer_head *bh,
  393. bh_end_io_t *handler)
  394. {
  395. bh->b_end_io = handler;
  396. set_buffer_async_write(bh);
  397. }
  398. void mark_buffer_async_write(struct buffer_head *bh)
  399. {
  400. mark_buffer_async_write_endio(bh, end_buffer_async_write);
  401. }
  402. EXPORT_SYMBOL(mark_buffer_async_write);
  403. /*
  404. * fs/buffer.c contains helper functions for buffer-backed address space's
  405. * fsync functions. A common requirement for buffer-based filesystems is
  406. * that certain data from the backing blockdev needs to be written out for
  407. * a successful fsync(). For example, ext2 indirect blocks need to be
  408. * written back and waited upon before fsync() returns.
  409. *
  410. * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
  411. * inode_has_buffers() and invalidate_inode_buffers() are provided for the
  412. * management of a list of dependent buffers at ->i_mapping->private_list.
  413. *
  414. * Locking is a little subtle: try_to_free_buffers() will remove buffers
  415. * from their controlling inode's queue when they are being freed. But
  416. * try_to_free_buffers() will be operating against the *blockdev* mapping
  417. * at the time, not against the S_ISREG file which depends on those buffers.
  418. * So the locking for private_list is via the private_lock in the address_space
  419. * which backs the buffers. Which is different from the address_space
  420. * against which the buffers are listed. So for a particular address_space,
  421. * mapping->private_lock does *not* protect mapping->private_list! In fact,
  422. * mapping->private_list will always be protected by the backing blockdev's
  423. * ->private_lock.
  424. *
  425. * Which introduces a requirement: all buffers on an address_space's
  426. * ->private_list must be from the same address_space: the blockdev's.
  427. *
  428. * address_spaces which do not place buffers at ->private_list via these
  429. * utility functions are free to use private_lock and private_list for
  430. * whatever they want. The only requirement is that list_empty(private_list)
  431. * be true at clear_inode() time.
  432. *
  433. * FIXME: clear_inode should not call invalidate_inode_buffers(). The
  434. * filesystems should do that. invalidate_inode_buffers() should just go
  435. * BUG_ON(!list_empty).
  436. *
  437. * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
  438. * take an address_space, not an inode. And it should be called
  439. * mark_buffer_dirty_fsync() to clearly define why those buffers are being
  440. * queued up.
  441. *
  442. * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
  443. * list if it is already on a list. Because if the buffer is on a list,
  444. * it *must* already be on the right one. If not, the filesystem is being
  445. * silly. This will save a ton of locking. But first we have to ensure
  446. * that buffers are taken *off* the old inode's list when they are freed
  447. * (presumably in truncate). That requires careful auditing of all
  448. * filesystems (do it inside bforget()). It could also be done by bringing
  449. * b_inode back.
  450. */
  451. /*
  452. * The buffer's backing address_space's private_lock must be held
  453. */
  454. static void __remove_assoc_queue(struct buffer_head *bh)
  455. {
  456. list_del_init(&bh->b_assoc_buffers);
  457. WARN_ON(!bh->b_assoc_map);
  458. if (buffer_write_io_error(bh))
  459. set_bit(AS_EIO, &bh->b_assoc_map->flags);
  460. bh->b_assoc_map = NULL;
  461. }
  462. int inode_has_buffers(struct inode *inode)
  463. {
  464. return !list_empty(&inode->i_data.private_list);
  465. }
  466. /*
  467. * osync is designed to support O_SYNC io. It waits synchronously for
  468. * all already-submitted IO to complete, but does not queue any new
  469. * writes to the disk.
  470. *
  471. * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
  472. * you dirty the buffers, and then use osync_inode_buffers to wait for
  473. * completion. Any other dirty buffers which are not yet queued for
  474. * write will not be flushed to disk by the osync.
  475. */
  476. static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
  477. {
  478. struct buffer_head *bh;
  479. struct list_head *p;
  480. int err = 0;
  481. spin_lock(lock);
  482. repeat:
  483. list_for_each_prev(p, list) {
  484. bh = BH_ENTRY(p);
  485. if (buffer_locked(bh)) {
  486. get_bh(bh);
  487. spin_unlock(lock);
  488. wait_on_buffer(bh);
  489. if (!buffer_uptodate(bh))
  490. err = -EIO;
  491. brelse(bh);
  492. spin_lock(lock);
  493. goto repeat;
  494. }
  495. }
  496. spin_unlock(lock);
  497. return err;
  498. }
  499. static void do_thaw_one(struct super_block *sb, void *unused)
  500. {
  501. char b[BDEVNAME_SIZE];
  502. while (sb->s_bdev && !thaw_bdev(sb->s_bdev, sb))
  503. printk(KERN_WARNING "Emergency Thaw on %s\n",
  504. bdevname(sb->s_bdev, b));
  505. }
  506. static void do_thaw_all(struct work_struct *work)
  507. {
  508. iterate_supers(do_thaw_one, NULL);
  509. kfree(work);
  510. printk(KERN_WARNING "Emergency Thaw complete\n");
  511. }
  512. /**
  513. * emergency_thaw_all -- forcibly thaw every frozen filesystem
  514. *
  515. * Used for emergency unfreeze of all filesystems via SysRq
  516. */
  517. void emergency_thaw_all(void)
  518. {
  519. struct work_struct *work;
  520. work = kmalloc(sizeof(*work), GFP_ATOMIC);
  521. if (work) {
  522. INIT_WORK(work, do_thaw_all);
  523. schedule_work(work);
  524. }
  525. }
  526. /**
  527. * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
  528. * @mapping: the mapping which wants those buffers written
  529. *
  530. * Starts I/O against the buffers at mapping->private_list, and waits upon
  531. * that I/O.
  532. *
  533. * Basically, this is a convenience function for fsync().
  534. * @mapping is a file or directory which needs those buffers to be written for
  535. * a successful fsync().
  536. */
  537. int sync_mapping_buffers(struct address_space *mapping)
  538. {
  539. struct address_space *buffer_mapping = mapping->private_data;
  540. if (buffer_mapping == NULL || list_empty(&mapping->private_list))
  541. return 0;
  542. return fsync_buffers_list(&buffer_mapping->private_lock,
  543. &mapping->private_list);
  544. }
  545. EXPORT_SYMBOL(sync_mapping_buffers);
  546. /*
  547. * Called when we've recently written block `bblock', and it is known that
  548. * `bblock' was for a buffer_boundary() buffer. This means that the block at
  549. * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
  550. * dirty, schedule it for IO. So that indirects merge nicely with their data.
  551. */
  552. void write_boundary_block(struct block_device *bdev,
  553. sector_t bblock, unsigned blocksize)
  554. {
  555. struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
  556. if (bh) {
  557. if (buffer_dirty(bh))
  558. ll_rw_block(WRITE, 1, &bh);
  559. put_bh(bh);
  560. }
  561. }
  562. void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
  563. {
  564. struct address_space *mapping = inode->i_mapping;
  565. struct address_space *buffer_mapping = bh->b_page->mapping;
  566. mark_buffer_dirty(bh);
  567. if (!mapping->private_data) {
  568. mapping->private_data = buffer_mapping;
  569. } else {
  570. BUG_ON(mapping->private_data != buffer_mapping);
  571. }
  572. if (!bh->b_assoc_map) {
  573. spin_lock(&buffer_mapping->private_lock);
  574. list_move_tail(&bh->b_assoc_buffers,
  575. &mapping->private_list);
  576. bh->b_assoc_map = mapping;
  577. spin_unlock(&buffer_mapping->private_lock);
  578. }
  579. }
  580. EXPORT_SYMBOL(mark_buffer_dirty_inode);
  581. /*
  582. * Mark the page dirty, and set it dirty in the radix tree, and mark the inode
  583. * dirty.
  584. *
  585. * If warn is true, then emit a warning if the page is not uptodate and has
  586. * not been truncated.
  587. */
  588. static void __set_page_dirty(struct page *page,
  589. struct address_space *mapping, int warn)
  590. {
  591. spin_lock_irq(&mapping->tree_lock);
  592. if (page->mapping) { /* Race with truncate? */
  593. WARN_ON_ONCE(warn && !PageUptodate(page));
  594. account_page_dirtied(page, mapping);
  595. radix_tree_tag_set(&mapping->page_tree,
  596. page_index(page), PAGECACHE_TAG_DIRTY);
  597. }
  598. spin_unlock_irq(&mapping->tree_lock);
  599. __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
  600. }
  601. /*
  602. * Add a page to the dirty page list.
  603. *
  604. * It is a sad fact of life that this function is called from several places
  605. * deeply under spinlocking. It may not sleep.
  606. *
  607. * If the page has buffers, the uptodate buffers are set dirty, to preserve
  608. * dirty-state coherency between the page and the buffers. It the page does
  609. * not have buffers then when they are later attached they will all be set
  610. * dirty.
  611. *
  612. * The buffers are dirtied before the page is dirtied. There's a small race
  613. * window in which a writepage caller may see the page cleanness but not the
  614. * buffer dirtiness. That's fine. If this code were to set the page dirty
  615. * before the buffers, a concurrent writepage caller could clear the page dirty
  616. * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
  617. * page on the dirty page list.
  618. *
  619. * We use private_lock to lock against try_to_free_buffers while using the
  620. * page's buffer list. Also use this to protect against clean buffers being
  621. * added to the page after it was set dirty.
  622. *
  623. * FIXME: may need to call ->reservepage here as well. That's rather up to the
  624. * address_space though.
  625. */
  626. int __set_page_dirty_buffers(struct page *page)
  627. {
  628. int newly_dirty;
  629. struct address_space *mapping = page_mapping(page);
  630. if (unlikely(!mapping))
  631. return !TestSetPageDirty(page);
  632. spin_lock(&mapping->private_lock);
  633. if (page_has_buffers(page)) {
  634. struct buffer_head *head = page_buffers(page);
  635. struct buffer_head *bh = head;
  636. do {
  637. set_buffer_dirty(bh);
  638. bh = bh->b_this_page;
  639. } while (bh != head);
  640. }
  641. newly_dirty = !TestSetPageDirty(page);
  642. spin_unlock(&mapping->private_lock);
  643. if (newly_dirty)
  644. __set_page_dirty(page, mapping, 1);
  645. return newly_dirty;
  646. }
  647. EXPORT_SYMBOL(__set_page_dirty_buffers);
  648. /*
  649. * Write out and wait upon a list of buffers.
  650. *
  651. * We have conflicting pressures: we want to make sure that all
  652. * initially dirty buffers get waited on, but that any subsequently
  653. * dirtied buffers don't. After all, we don't want fsync to last
  654. * forever if somebody is actively writing to the file.
  655. *
  656. * Do this in two main stages: first we copy dirty buffers to a
  657. * temporary inode list, queueing the writes as we go. Then we clean
  658. * up, waiting for those writes to complete.
  659. *
  660. * During this second stage, any subsequent updates to the file may end
  661. * up refiling the buffer on the original inode's dirty list again, so
  662. * there is a chance we will end up with a buffer queued for write but
  663. * not yet completed on that list. So, as a final cleanup we go through
  664. * the osync code to catch these locked, dirty buffers without requeuing
  665. * any newly dirty buffers for write.
  666. */
  667. static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
  668. {
  669. struct buffer_head *bh;
  670. struct list_head tmp;
  671. struct address_space *mapping;
  672. int err = 0, err2;
  673. struct blk_plug plug;
  674. INIT_LIST_HEAD(&tmp);
  675. blk_start_plug(&plug);
  676. spin_lock(lock);
  677. while (!list_empty(list)) {
  678. bh = BH_ENTRY(list->next);
  679. mapping = bh->b_assoc_map;
  680. __remove_assoc_queue(bh);
  681. /* Avoid race with mark_buffer_dirty_inode() which does
  682. * a lockless check and we rely on seeing the dirty bit */
  683. smp_mb();
  684. if (buffer_dirty(bh) || buffer_locked(bh)) {
  685. list_add(&bh->b_assoc_buffers, &tmp);
  686. bh->b_assoc_map = mapping;
  687. if (buffer_dirty(bh)) {
  688. get_bh(bh);
  689. spin_unlock(lock);
  690. /*
  691. * Ensure any pending I/O completes so that
  692. * write_dirty_buffer() actually writes the
  693. * current contents - it is a noop if I/O is
  694. * still in flight on potentially older
  695. * contents.
  696. */
  697. write_dirty_buffer(bh, WRITE_SYNC);
  698. /*
  699. * Kick off IO for the previous mapping. Note
  700. * that we will not run the very last mapping,
  701. * wait_on_buffer() will do that for us
  702. * through sync_buffer().
  703. */
  704. brelse(bh);
  705. spin_lock(lock);
  706. }
  707. }
  708. }
  709. spin_unlock(lock);
  710. blk_finish_plug(&plug);
  711. spin_lock(lock);
  712. while (!list_empty(&tmp)) {
  713. bh = BH_ENTRY(tmp.prev);
  714. get_bh(bh);
  715. mapping = bh->b_assoc_map;
  716. __remove_assoc_queue(bh);
  717. /* Avoid race with mark_buffer_dirty_inode() which does
  718. * a lockless check and we rely on seeing the dirty bit */
  719. smp_mb();
  720. if (buffer_dirty(bh)) {
  721. list_add(&bh->b_assoc_buffers,
  722. &mapping->private_list);
  723. bh->b_assoc_map = mapping;
  724. }
  725. spin_unlock(lock);
  726. wait_on_buffer(bh);
  727. if (!buffer_uptodate(bh))
  728. err = -EIO;
  729. brelse(bh);
  730. spin_lock(lock);
  731. }
  732. spin_unlock(lock);
  733. err2 = osync_buffers_list(lock, list);
  734. if (err)
  735. return err;
  736. else
  737. return err2;
  738. }
  739. /*
  740. * Invalidate any and all dirty buffers on a given inode. We are
  741. * probably unmounting the fs, but that doesn't mean we have already
  742. * done a sync(). Just drop the buffers from the inode list.
  743. *
  744. * NOTE: we take the inode's blockdev's mapping's private_lock. Which
  745. * assumes that all the buffers are against the blockdev. Not true
  746. * for reiserfs.
  747. */
  748. void invalidate_inode_buffers(struct inode *inode)
  749. {
  750. if (inode_has_buffers(inode)) {
  751. struct address_space *mapping = &inode->i_data;
  752. struct list_head *list = &mapping->private_list;
  753. struct address_space *buffer_mapping = mapping->private_data;
  754. spin_lock(&buffer_mapping->private_lock);
  755. while (!list_empty(list))
  756. __remove_assoc_queue(BH_ENTRY(list->next));
  757. spin_unlock(&buffer_mapping->private_lock);
  758. }
  759. }
  760. EXPORT_SYMBOL(invalidate_inode_buffers);
  761. /*
  762. * Remove any clean buffers from the inode's buffer list. This is called
  763. * when we're trying to free the inode itself. Those buffers can pin it.
  764. *
  765. * Returns true if all buffers were removed.
  766. */
  767. int remove_inode_buffers(struct inode *inode)
  768. {
  769. int ret = 1;
  770. if (inode_has_buffers(inode)) {
  771. struct address_space *mapping = &inode->i_data;
  772. struct list_head *list = &mapping->private_list;
  773. struct address_space *buffer_mapping = mapping->private_data;
  774. spin_lock(&buffer_mapping->private_lock);
  775. while (!list_empty(list)) {
  776. struct buffer_head *bh = BH_ENTRY(list->next);
  777. if (buffer_dirty(bh)) {
  778. ret = 0;
  779. break;
  780. }
  781. __remove_assoc_queue(bh);
  782. }
  783. spin_unlock(&buffer_mapping->private_lock);
  784. }
  785. return ret;
  786. }
  787. /*
  788. * Create the appropriate buffers when given a page for data area and
  789. * the size of each buffer.. Use the bh->b_this_page linked list to
  790. * follow the buffers created. Return NULL if unable to create more
  791. * buffers.
  792. *
  793. * The retry flag is used to differentiate async IO (paging, swapping)
  794. * which may not fail from ordinary buffer allocations.
  795. */
  796. struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
  797. int retry)
  798. {
  799. struct buffer_head *bh, *head;
  800. long offset;
  801. try_again:
  802. head = NULL;
  803. offset = PAGE_SIZE;
  804. while ((offset -= size) >= 0) {
  805. bh = alloc_buffer_head(GFP_NOFS);
  806. if (!bh)
  807. goto no_grow;
  808. bh->b_this_page = head;
  809. bh->b_blocknr = -1;
  810. head = bh;
  811. bh->b_size = size;
  812. /* Link the buffer to its page */
  813. set_bh_page(bh, page, offset);
  814. }
  815. return head;
  816. /*
  817. * In case anything failed, we just free everything we got.
  818. */
  819. no_grow:
  820. if (head) {
  821. do {
  822. bh = head;
  823. head = head->b_this_page;
  824. free_buffer_head(bh);
  825. } while (head);
  826. }
  827. /*
  828. * Return failure for non-async IO requests. Async IO requests
  829. * are not allowed to fail, so we have to wait until buffer heads
  830. * become available. But we don't want tasks sleeping with
  831. * partially complete buffers, so all were released above.
  832. */
  833. if (!retry)
  834. return NULL;
  835. /* We're _really_ low on memory. Now we just
  836. * wait for old buffer heads to become free due to
  837. * finishing IO. Since this is an async request and
  838. * the reserve list is empty, we're sure there are
  839. * async buffer heads in use.
  840. */
  841. free_more_memory();
  842. goto try_again;
  843. }
  844. EXPORT_SYMBOL_GPL(alloc_page_buffers);
  845. static inline void
  846. link_dev_buffers(struct page *page, struct buffer_head *head)
  847. {
  848. struct buffer_head *bh, *tail;
  849. bh = head;
  850. do {
  851. tail = bh;
  852. bh = bh->b_this_page;
  853. } while (bh);
  854. tail->b_this_page = head;
  855. attach_page_buffers(page, head);
  856. }
  857. static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size)
  858. {
  859. sector_t retval = ~((sector_t)0);
  860. loff_t sz = i_size_read(bdev->bd_inode);
  861. if (sz) {
  862. unsigned int sizebits = blksize_bits(size);
  863. retval = (sz >> sizebits);
  864. }
  865. return retval;
  866. }
  867. /*
  868. * Initialise the state of a blockdev page's buffers.
  869. */
  870. static sector_t
  871. init_page_buffers(struct page *page, struct block_device *bdev,
  872. sector_t block, int size)
  873. {
  874. struct buffer_head *head = page_buffers(page);
  875. struct buffer_head *bh = head;
  876. int uptodate = PageUptodate(page);
  877. sector_t end_block = blkdev_max_block(I_BDEV(bdev->bd_inode), size);
  878. do {
  879. if (!buffer_mapped(bh)) {
  880. init_buffer(bh, NULL, NULL);
  881. bh->b_bdev = bdev;
  882. bh->b_blocknr = block;
  883. if (uptodate)
  884. set_buffer_uptodate(bh);
  885. if (block < end_block)
  886. set_buffer_mapped(bh);
  887. }
  888. block++;
  889. bh = bh->b_this_page;
  890. } while (bh != head);
  891. /*
  892. * Caller needs to validate requested block against end of device.
  893. */
  894. return end_block;
  895. }
  896. /*
  897. * Create the page-cache page that contains the requested block.
  898. *
  899. * This is used purely for blockdev mappings.
  900. */
  901. static int
  902. grow_dev_page(struct block_device *bdev, sector_t block,
  903. pgoff_t index, int size, int sizebits)
  904. {
  905. struct inode *inode = bdev->bd_inode;
  906. struct page *page;
  907. struct buffer_head *bh;
  908. sector_t end_block;
  909. int ret = 0; /* Will call free_more_memory() */
  910. gfp_t gfp_mask;
  911. gfp_mask = mapping_gfp_mask(inode->i_mapping) & ~__GFP_FS;
  912. gfp_mask |= __GFP_MOVABLE;
  913. /*
  914. * XXX: __getblk_slow() can not really deal with failure and
  915. * will endlessly loop on improvised global reclaim. Prefer
  916. * looping in the allocator rather than here, at least that
  917. * code knows what it's doing.
  918. */
  919. gfp_mask |= __GFP_NOFAIL;
  920. page = find_or_create_page(inode->i_mapping, index, gfp_mask);
  921. if (!page)
  922. return ret;
  923. BUG_ON(!PageLocked(page));
  924. if (page_has_buffers(page)) {
  925. bh = page_buffers(page);
  926. if (bh->b_size == size) {
  927. end_block = init_page_buffers(page, bdev,
  928. index << sizebits, size);
  929. goto done;
  930. }
  931. if (!try_to_free_buffers(page))
  932. goto failed;
  933. }
  934. /*
  935. * Allocate some buffers for this page
  936. */
  937. bh = alloc_page_buffers(page, size, 0);
  938. if (!bh)
  939. goto failed;
  940. /*
  941. * Link the page to the buffers and initialise them. Take the
  942. * lock to be atomic wrt __find_get_block(), which does not
  943. * run under the page lock.
  944. */
  945. spin_lock(&inode->i_mapping->private_lock);
  946. link_dev_buffers(page, bh);
  947. end_block = init_page_buffers(page, bdev, index << sizebits, size);
  948. spin_unlock(&inode->i_mapping->private_lock);
  949. done:
  950. ret = (block < end_block) ? 1 : -ENXIO;
  951. failed:
  952. unlock_page(page);
  953. page_cache_release(page);
  954. return ret;
  955. }
  956. /*
  957. * Create buffers for the specified block device block's page. If
  958. * that page was dirty, the buffers are set dirty also.
  959. */
  960. static int
  961. grow_buffers(struct block_device *bdev, sector_t block, int size)
  962. {
  963. pgoff_t index;
  964. int sizebits;
  965. sizebits = -1;
  966. do {
  967. sizebits++;
  968. } while ((size << sizebits) < PAGE_SIZE);
  969. index = block >> sizebits;
  970. /*
  971. * Check for a block which wants to lie outside our maximum possible
  972. * pagecache index. (this comparison is done using sector_t types).
  973. */
  974. if (unlikely(index != block >> sizebits)) {
  975. char b[BDEVNAME_SIZE];
  976. printk(KERN_ERR "%s: requested out-of-range block %llu for "
  977. "device %s\n",
  978. __func__, (unsigned long long)block,
  979. bdevname(bdev, b));
  980. return -EIO;
  981. }
  982. /* Create a page with the proper size buffers.. */
  983. return grow_dev_page(bdev, block, index, size, sizebits);
  984. }
  985. static struct buffer_head *
  986. __getblk_slow(struct block_device *bdev, sector_t block, int size)
  987. {
  988. /* Size must be multiple of hard sectorsize */
  989. if (unlikely(size & (bdev_logical_block_size(bdev)-1) ||
  990. (size < 512 || size > PAGE_SIZE))) {
  991. printk(KERN_ERR "getblk(): invalid block size %d requested\n",
  992. size);
  993. printk(KERN_ERR "logical block size: %d\n",
  994. bdev_logical_block_size(bdev));
  995. dump_stack();
  996. return NULL;
  997. }
  998. for (;;) {
  999. struct buffer_head *bh;
  1000. int ret;
  1001. bh = __find_get_block(bdev, block, size);
  1002. if (bh)
  1003. return bh;
  1004. ret = grow_buffers(bdev, block, size);
  1005. if (ret < 0)
  1006. return NULL;
  1007. if (ret == 0)
  1008. free_more_memory();
  1009. }
  1010. }
  1011. /*
  1012. * The relationship between dirty buffers and dirty pages:
  1013. *
  1014. * Whenever a page has any dirty buffers, the page's dirty bit is set, and
  1015. * the page is tagged dirty in its radix tree.
  1016. *
  1017. * At all times, the dirtiness of the buffers represents the dirtiness of
  1018. * subsections of the page. If the page has buffers, the page dirty bit is
  1019. * merely a hint about the true dirty state.
  1020. *
  1021. * When a page is set dirty in its entirety, all its buffers are marked dirty
  1022. * (if the page has buffers).
  1023. *
  1024. * When a buffer is marked dirty, its page is dirtied, but the page's other
  1025. * buffers are not.
  1026. *
  1027. * Also. When blockdev buffers are explicitly read with bread(), they
  1028. * individually become uptodate. But their backing page remains not
  1029. * uptodate - even if all of its buffers are uptodate. A subsequent
  1030. * block_read_full_page() against that page will discover all the uptodate
  1031. * buffers, will set the page uptodate and will perform no I/O.
  1032. */
  1033. /**
  1034. * mark_buffer_dirty - mark a buffer_head as needing writeout
  1035. * @bh: the buffer_head to mark dirty
  1036. *
  1037. * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
  1038. * backing page dirty, then tag the page as dirty in its address_space's radix
  1039. * tree and then attach the address_space's inode to its superblock's dirty
  1040. * inode list.
  1041. *
  1042. * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock,
  1043. * mapping->tree_lock and mapping->host->i_lock.
  1044. */
  1045. void mark_buffer_dirty(struct buffer_head *bh)
  1046. {
  1047. WARN_ON_ONCE(!buffer_uptodate(bh));
  1048. trace_block_dirty_buffer(bh);
  1049. /*
  1050. * Very *carefully* optimize the it-is-already-dirty case.
  1051. *
  1052. * Don't let the final "is it dirty" escape to before we
  1053. * perhaps modified the buffer.
  1054. */
  1055. if (buffer_dirty(bh)) {
  1056. smp_mb();
  1057. if (buffer_dirty(bh))
  1058. return;
  1059. }
  1060. if (!test_set_buffer_dirty(bh)) {
  1061. struct page *page = bh->b_page;
  1062. if (!TestSetPageDirty(page)) {
  1063. struct address_space *mapping = page_mapping(page);
  1064. if (mapping)
  1065. __set_page_dirty(page, mapping, 0);
  1066. }
  1067. }
  1068. }
  1069. EXPORT_SYMBOL(mark_buffer_dirty);
  1070. /*
  1071. * Decrement a buffer_head's reference count. If all buffers against a page
  1072. * have zero reference count, are clean and unlocked, and if the page is clean
  1073. * and unlocked then try_to_free_buffers() may strip the buffers from the page
  1074. * in preparation for freeing it (sometimes, rarely, buffers are removed from
  1075. * a page but it ends up not being freed, and buffers may later be reattached).
  1076. */
  1077. void __brelse(struct buffer_head * buf)
  1078. {
  1079. if (atomic_read(&buf->b_count)) {
  1080. put_bh(buf);
  1081. return;
  1082. }
  1083. WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
  1084. }
  1085. EXPORT_SYMBOL(__brelse);
  1086. /*
  1087. * bforget() is like brelse(), except it discards any
  1088. * potentially dirty data.
  1089. */
  1090. void __bforget(struct buffer_head *bh)
  1091. {
  1092. clear_buffer_dirty(bh);
  1093. if (bh->b_assoc_map) {
  1094. struct address_space *buffer_mapping = bh->b_page->mapping;
  1095. spin_lock(&buffer_mapping->private_lock);
  1096. list_del_init(&bh->b_assoc_buffers);
  1097. bh->b_assoc_map = NULL;
  1098. spin_unlock(&buffer_mapping->private_lock);
  1099. }
  1100. __brelse(bh);
  1101. }
  1102. EXPORT_SYMBOL(__bforget);
  1103. static struct buffer_head *__bread_slow(struct buffer_head *bh)
  1104. {
  1105. lock_buffer(bh);
  1106. if (buffer_uptodate(bh)) {
  1107. unlock_buffer(bh);
  1108. return bh;
  1109. } else {
  1110. get_bh(bh);
  1111. bh->b_end_io = end_buffer_read_sync;
  1112. submit_bh(READ, bh);
  1113. wait_on_buffer(bh);
  1114. if (buffer_uptodate(bh))
  1115. return bh;
  1116. }
  1117. brelse(bh);
  1118. return NULL;
  1119. }
  1120. /*
  1121. * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
  1122. * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
  1123. * refcount elevated by one when they're in an LRU. A buffer can only appear
  1124. * once in a particular CPU's LRU. A single buffer can be present in multiple
  1125. * CPU's LRUs at the same time.
  1126. *
  1127. * This is a transparent caching front-end to sb_bread(), sb_getblk() and
  1128. * sb_find_get_block().
  1129. *
  1130. * The LRUs themselves only need locking against invalidate_bh_lrus. We use
  1131. * a local interrupt disable for that.
  1132. */
  1133. #define BH_LRU_SIZE 8
  1134. struct bh_lru {
  1135. struct buffer_head *bhs[BH_LRU_SIZE];
  1136. };
  1137. static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
  1138. #ifdef CONFIG_SMP
  1139. #define bh_lru_lock() local_irq_disable()
  1140. #define bh_lru_unlock() local_irq_enable()
  1141. #else
  1142. #define bh_lru_lock() preempt_disable()
  1143. #define bh_lru_unlock() preempt_enable()
  1144. #endif
  1145. static inline void check_irqs_on(void)
  1146. {
  1147. #ifdef irqs_disabled
  1148. BUG_ON(irqs_disabled());
  1149. #endif
  1150. }
  1151. /*
  1152. * The LRU management algorithm is dopey-but-simple. Sorry.
  1153. */
  1154. static void bh_lru_install(struct buffer_head *bh)
  1155. {
  1156. struct buffer_head *evictee = NULL;
  1157. check_irqs_on();
  1158. bh_lru_lock();
  1159. if (__this_cpu_read(bh_lrus.bhs[0]) != bh) {
  1160. struct buffer_head *bhs[BH_LRU_SIZE];
  1161. int in;
  1162. int out = 0;
  1163. get_bh(bh);
  1164. bhs[out++] = bh;
  1165. for (in = 0; in < BH_LRU_SIZE; in++) {
  1166. struct buffer_head *bh2 =
  1167. __this_cpu_read(bh_lrus.bhs[in]);
  1168. if (bh2 == bh) {
  1169. __brelse(bh2);
  1170. } else {
  1171. if (out >= BH_LRU_SIZE) {
  1172. BUG_ON(evictee != NULL);
  1173. evictee = bh2;
  1174. } else {
  1175. bhs[out++] = bh2;
  1176. }
  1177. }
  1178. }
  1179. while (out < BH_LRU_SIZE)
  1180. bhs[out++] = NULL;
  1181. memcpy(__this_cpu_ptr(&bh_lrus.bhs), bhs, sizeof(bhs));
  1182. }
  1183. bh_lru_unlock();
  1184. if (evictee)
  1185. __brelse(evictee);
  1186. }
  1187. /*
  1188. * Look up the bh in this cpu's LRU. If it's there, move it to the head.
  1189. */
  1190. static struct buffer_head *
  1191. lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
  1192. {
  1193. struct buffer_head *ret = NULL;
  1194. unsigned int i;
  1195. check_irqs_on();
  1196. bh_lru_lock();
  1197. for (i = 0; i < BH_LRU_SIZE; i++) {
  1198. struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]);
  1199. if (bh && bh->b_bdev == bdev &&
  1200. bh->b_blocknr == block && bh->b_size == size) {
  1201. if (i) {
  1202. while (i) {
  1203. __this_cpu_write(bh_lrus.bhs[i],
  1204. __this_cpu_read(bh_lrus.bhs[i - 1]));
  1205. i--;
  1206. }
  1207. __this_cpu_write(bh_lrus.bhs[0], bh);
  1208. }
  1209. get_bh(bh);
  1210. ret = bh;
  1211. break;
  1212. }
  1213. }
  1214. bh_lru_unlock();
  1215. return ret;
  1216. }
  1217. /*
  1218. * Perform a pagecache lookup for the matching buffer. If it's there, refresh
  1219. * it in the LRU and mark it as accessed. If it is not present then return
  1220. * NULL
  1221. */
  1222. struct buffer_head *
  1223. __find_get_block(struct block_device *bdev, sector_t block, unsigned size)
  1224. {
  1225. struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
  1226. if (bh == NULL) {
  1227. bh = __find_get_block_slow(bdev, block);
  1228. if (bh)
  1229. bh_lru_install(bh);
  1230. }
  1231. if (bh)
  1232. touch_buffer(bh);
  1233. return bh;
  1234. }
  1235. EXPORT_SYMBOL(__find_get_block);
  1236. /*
  1237. * __getblk will locate (and, if necessary, create) the buffer_head
  1238. * which corresponds to the passed block_device, block and size. The
  1239. * returned buffer has its reference count incremented.
  1240. *
  1241. * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
  1242. * attempt is failing. FIXME, perhaps?
  1243. */
  1244. struct buffer_head *
  1245. __getblk(struct block_device *bdev, sector_t block, unsigned size)
  1246. {
  1247. struct buffer_head *bh = __find_get_block(bdev, block, size);
  1248. might_sleep();
  1249. if (bh == NULL)
  1250. bh = __getblk_slow(bdev, block, size);
  1251. return bh;
  1252. }
  1253. EXPORT_SYMBOL(__getblk);
  1254. /*
  1255. * Do async read-ahead on a buffer..
  1256. */
  1257. void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
  1258. {
  1259. struct buffer_head *bh = __getblk(bdev, block, size);
  1260. if (likely(bh)) {
  1261. ll_rw_block(READA, 1, &bh);
  1262. brelse(bh);
  1263. }
  1264. }
  1265. EXPORT_SYMBOL(__breadahead);
  1266. /**
  1267. * __bread() - reads a specified block and returns the bh
  1268. * @bdev: the block_device to read from
  1269. * @block: number of block
  1270. * @size: size (in bytes) to read
  1271. *
  1272. * Reads a specified block, and returns buffer head that contains it.
  1273. * It returns NULL if the block was unreadable.
  1274. */
  1275. struct buffer_head *
  1276. __bread(struct block_device *bdev, sector_t block, unsigned size)
  1277. {
  1278. struct buffer_head *bh = __getblk(bdev, block, size);
  1279. if (likely(bh) && !buffer_uptodate(bh))
  1280. bh = __bread_slow(bh);
  1281. return bh;
  1282. }
  1283. EXPORT_SYMBOL(__bread);
  1284. /*
  1285. * invalidate_bh_lrus() is called rarely - but not only at unmount.
  1286. * This doesn't race because it runs in each cpu either in irq
  1287. * or with preempt disabled.
  1288. */
  1289. static void invalidate_bh_lru(void *arg)
  1290. {
  1291. struct bh_lru *b = &get_cpu_var(bh_lrus);
  1292. int i;
  1293. for (i = 0; i < BH_LRU_SIZE; i++) {
  1294. brelse(b->bhs[i]);
  1295. b->bhs[i] = NULL;
  1296. }
  1297. put_cpu_var(bh_lrus);
  1298. }
  1299. static bool has_bh_in_lru(int cpu, void *dummy)
  1300. {
  1301. struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu);
  1302. int i;
  1303. for (i = 0; i < BH_LRU_SIZE; i++) {
  1304. if (b->bhs[i])
  1305. return 1;
  1306. }
  1307. return 0;
  1308. }
  1309. void invalidate_bh_lrus(void)
  1310. {
  1311. on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1, GFP_KERNEL);
  1312. }
  1313. EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
  1314. void set_bh_page(struct buffer_head *bh,
  1315. struct page *page, unsigned long offset)
  1316. {
  1317. bh->b_page = page;
  1318. BUG_ON(offset >= PAGE_SIZE);
  1319. if (PageHighMem(page))
  1320. /*
  1321. * This catches illegal uses and preserves the offset:
  1322. */
  1323. bh->b_data = (char *)(0 + offset);
  1324. else
  1325. bh->b_data = page_address(page) + offset;
  1326. }
  1327. EXPORT_SYMBOL(set_bh_page);
  1328. /*
  1329. * Called when truncating a buffer on a page completely.
  1330. */
  1331. static void discard_buffer(struct buffer_head * bh)
  1332. {
  1333. lock_buffer(bh);
  1334. clear_buffer_dirty(bh);
  1335. bh->b_bdev = NULL;
  1336. clear_buffer_mapped(bh);
  1337. clear_buffer_req(bh);
  1338. clear_buffer_new(bh);
  1339. clear_buffer_delay(bh);
  1340. clear_buffer_unwritten(bh);
  1341. unlock_buffer(bh);
  1342. }
  1343. /**
  1344. * block_invalidatepage - invalidate part or all of a buffer-backed page
  1345. *
  1346. * @page: the page which is affected
  1347. * @offset: start of the range to invalidate
  1348. * @length: length of the range to invalidate
  1349. *
  1350. * block_invalidatepage() is called when all or part of the page has become
  1351. * invalidated by a truncate operation.
  1352. *
  1353. * block_invalidatepage() does not have to release all buffers, but it must
  1354. * ensure that no dirty buffer is left outside @offset and that no I/O
  1355. * is underway against any of the blocks which are outside the truncation
  1356. * point. Because the caller is about to free (and possibly reuse) those
  1357. * blocks on-disk.
  1358. */
  1359. void block_invalidatepage(struct page *page, unsigned int offset,
  1360. unsigned int length)
  1361. {
  1362. struct buffer_head *head, *bh, *next;
  1363. unsigned int curr_off = 0;
  1364. unsigned int stop = length + offset;
  1365. BUG_ON(!PageLocked(page));
  1366. if (!page_has_buffers(page))
  1367. goto out;
  1368. /*
  1369. * Check for overflow
  1370. */
  1371. BUG_ON(stop > PAGE_CACHE_SIZE || stop < length);
  1372. head = page_buffers(page);
  1373. bh = head;
  1374. do {
  1375. unsigned int next_off = curr_off + bh->b_size;
  1376. next = bh->b_this_page;
  1377. /*
  1378. * Are we still fully in range ?
  1379. */
  1380. if (next_off > stop)
  1381. goto out;
  1382. /*
  1383. * is this block fully invalidated?
  1384. */
  1385. if (offset <= curr_off)
  1386. discard_buffer(bh);
  1387. curr_off = next_off;
  1388. bh = next;
  1389. } while (bh != head);
  1390. /*
  1391. * We release buffers only if the entire page is being invalidated.
  1392. * The get_block cached value has been unconditionally invalidated,
  1393. * so real IO is not possible anymore.
  1394. */
  1395. if (offset == 0)
  1396. try_to_release_page(page, 0);
  1397. out:
  1398. return;
  1399. }
  1400. EXPORT_SYMBOL(block_invalidatepage);
  1401. /*
  1402. * We attach and possibly dirty the buffers atomically wrt
  1403. * __set_page_dirty_buffers() via private_lock. try_to_free_buffers
  1404. * is already excluded via the page lock.
  1405. */
  1406. void create_empty_buffers(struct page *page,
  1407. unsigned long blocksize, unsigned long b_state)
  1408. {
  1409. struct buffer_head *bh, *head, *tail;
  1410. head = alloc_page_buffers(page, blocksize, 1);
  1411. bh = head;
  1412. do {
  1413. bh->b_state |= b_state;
  1414. tail = bh;
  1415. bh = bh->b_this_page;
  1416. } while (bh);
  1417. tail->b_this_page = head;
  1418. spin_lock(&page->mapping->private_lock);
  1419. if (PageUptodate(page) || PageDirty(page)) {
  1420. bh = head;
  1421. do {
  1422. if (PageDirty(page))
  1423. set_buffer_dirty(bh);
  1424. if (PageUptodate(page))
  1425. set_buffer_uptodate(bh);
  1426. bh = bh->b_this_page;
  1427. } while (bh != head);
  1428. }
  1429. attach_page_buffers(page, head);
  1430. spin_unlock(&page->mapping->private_lock);
  1431. }
  1432. EXPORT_SYMBOL(create_empty_buffers);
  1433. /*
  1434. * We are taking a block for data and we don't want any output from any
  1435. * buffer-cache aliases starting from return from that function and
  1436. * until the moment when something will explicitly mark the buffer
  1437. * dirty (hopefully that will not happen until we will free that block ;-)
  1438. * We don't even need to mark it not-uptodate - nobody can expect
  1439. * anything from a newly allocated buffer anyway. We used to used
  1440. * unmap_buffer() for such invalidation, but that was wrong. We definitely
  1441. * don't want to mark the alias unmapped, for example - it would confuse
  1442. * anyone who might pick it with bread() afterwards...
  1443. *
  1444. * Also.. Note that bforget() doesn't lock the buffer. So there can
  1445. * be writeout I/O going on against recently-freed buffers. We don't
  1446. * wait on that I/O in bforget() - it's more efficient to wait on the I/O
  1447. * only if we really need to. That happens here.
  1448. */
  1449. void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
  1450. {
  1451. struct buffer_head *old_bh;
  1452. might_sleep();
  1453. old_bh = __find_get_block_slow(bdev, block);
  1454. if (old_bh) {
  1455. clear_buffer_dirty(old_bh);
  1456. wait_on_buffer(old_bh);
  1457. clear_buffer_req(old_bh);
  1458. __brelse(old_bh);
  1459. }
  1460. }
  1461. EXPORT_SYMBOL(unmap_underlying_metadata);
  1462. /*
  1463. * Size is a power-of-two in the range 512..PAGE_SIZE,
  1464. * and the case we care about most is PAGE_SIZE.
  1465. *
  1466. * So this *could* possibly be written with those
  1467. * constraints in mind (relevant mostly if some
  1468. * architecture has a slow bit-scan instruction)
  1469. */
  1470. static inline int block_size_bits(unsigned int blocksize)
  1471. {
  1472. return ilog2(blocksize);
  1473. }
  1474. static struct buffer_head *create_page_buffers(struct page *page, struct inode *inode, unsigned int b_state)
  1475. {
  1476. BUG_ON(!PageLocked(page));
  1477. if (!page_has_buffers(page))
  1478. create_empty_buffers(page, 1 << ACCESS_ONCE(inode->i_blkbits), b_state);
  1479. return page_buffers(page);
  1480. }
  1481. /*
  1482. * NOTE! All mapped/uptodate combinations are valid:
  1483. *
  1484. * Mapped Uptodate Meaning
  1485. *
  1486. * No No "unknown" - must do get_block()
  1487. * No Yes "hole" - zero-filled
  1488. * Yes No "allocated" - allocated on disk, not read in
  1489. * Yes Yes "valid" - allocated and up-to-date in memory.
  1490. *
  1491. * "Dirty" is valid only with the last case (mapped+uptodate).
  1492. */
  1493. /*
  1494. * While block_write_full_page is writing back the dirty buffers under
  1495. * the page lock, whoever dirtied the buffers may decide to clean them
  1496. * again at any time. We handle that by only looking at the buffer
  1497. * state inside lock_buffer().
  1498. *
  1499. * If block_write_full_page() is called for regular writeback
  1500. * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
  1501. * locked buffer. This only can happen if someone has written the buffer
  1502. * directly, with submit_bh(). At the address_space level PageWriteback
  1503. * prevents this contention from occurring.
  1504. *
  1505. * If block_write_full_page() is called with wbc->sync_mode ==
  1506. * WB_SYNC_ALL, the writes are posted using WRITE_SYNC; this
  1507. * causes the writes to be flagged as synchronous writes.
  1508. */
  1509. static int __block_write_full_page(struct inode *inode, struct page *page,
  1510. get_block_t *get_block, struct writeback_control *wbc,
  1511. bh_end_io_t *handler)
  1512. {
  1513. int err;
  1514. sector_t block;
  1515. sector_t last_block;
  1516. struct buffer_head *bh, *head;
  1517. unsigned int blocksize, bbits;
  1518. int nr_underway = 0;
  1519. int write_op = (wbc->sync_mode == WB_SYNC_ALL ?
  1520. WRITE_SYNC : WRITE);
  1521. head = create_page_buffers(page, inode,
  1522. (1 << BH_Dirty)|(1 << BH_Uptodate));
  1523. /*
  1524. * Be very careful. We have no exclusion from __set_page_dirty_buffers
  1525. * here, and the (potentially unmapped) buffers may become dirty at
  1526. * any time. If a buffer becomes dirty here after we've inspected it
  1527. * then we just miss that fact, and the page stays dirty.
  1528. *
  1529. * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
  1530. * handle that here by just cleaning them.
  1531. */
  1532. bh = head;
  1533. blocksize = bh->b_size;
  1534. bbits = block_size_bits(blocksize);
  1535. block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
  1536. last_block = (i_size_read(inode) - 1) >> bbits;
  1537. /*
  1538. * Get all the dirty buffers mapped to disk addresses and
  1539. * handle any aliases from the underlying blockdev's mapping.
  1540. */
  1541. do {
  1542. if (block > last_block) {
  1543. /*
  1544. * mapped buffers outside i_size will occur, because
  1545. * this page can be outside i_size when there is a
  1546. * truncate in progress.
  1547. */
  1548. /*
  1549. * The buffer was zeroed by block_write_full_page()
  1550. */
  1551. clear_buffer_dirty(bh);
  1552. set_buffer_uptodate(bh);
  1553. } else if ((!buffer_mapped(bh) || buffer_delay(bh)) &&
  1554. buffer_dirty(bh)) {
  1555. WARN_ON(bh->b_size != blocksize);
  1556. err = get_block(inode, block, bh, 1);
  1557. if (err)
  1558. goto recover;
  1559. clear_buffer_delay(bh);
  1560. if (buffer_new(bh)) {
  1561. /* blockdev mappings never come here */
  1562. clear_buffer_new(bh);
  1563. unmap_underlying_metadata(bh->b_bdev,
  1564. bh->b_blocknr);
  1565. }
  1566. }
  1567. bh = bh->b_this_page;
  1568. block++;
  1569. } while (bh != head);
  1570. do {
  1571. if (!buffer_mapped(bh))
  1572. continue;
  1573. /*
  1574. * If it's a fully non-blocking write attempt and we cannot
  1575. * lock the buffer then redirty the page. Note that this can
  1576. * potentially cause a busy-wait loop from writeback threads
  1577. * and kswapd activity, but those code paths have their own
  1578. * higher-level throttling.
  1579. */
  1580. if (wbc->sync_mode != WB_SYNC_NONE) {
  1581. lock_buffer(bh);
  1582. } else if (!trylock_buffer(bh)) {
  1583. redirty_page_for_writepage(wbc, page);
  1584. continue;
  1585. }
  1586. if (test_clear_buffer_dirty(bh)) {
  1587. mark_buffer_async_write_endio(bh, handler);
  1588. } else {
  1589. unlock_buffer(bh);
  1590. }
  1591. } while ((bh = bh->b_this_page) != head);
  1592. /*
  1593. * The page and its buffers are protected by PageWriteback(), so we can
  1594. * drop the bh refcounts early.
  1595. */
  1596. BUG_ON(PageWriteback(page));
  1597. set_page_writeback(page);
  1598. do {
  1599. struct buffer_head *next = bh->b_this_page;
  1600. if (buffer_async_write(bh)) {
  1601. submit_bh(write_op, bh);
  1602. nr_underway++;
  1603. }
  1604. bh = next;
  1605. } while (bh != head);
  1606. unlock_page(page);
  1607. err = 0;
  1608. done:
  1609. if (nr_underway == 0) {
  1610. /*
  1611. * The page was marked dirty, but the buffers were
  1612. * clean. Someone wrote them back by hand with
  1613. * ll_rw_block/submit_bh. A rare case.
  1614. */
  1615. end_page_writeback(page);
  1616. /*
  1617. * The page and buffer_heads can be released at any time from
  1618. * here on.
  1619. */
  1620. }
  1621. return err;
  1622. recover:
  1623. /*
  1624. * ENOSPC, or some other error. We may already have added some
  1625. * blocks to the file, so we need to write these out to avoid
  1626. * exposing stale data.
  1627. * The page is currently locked and not marked for writeback
  1628. */
  1629. bh = head;
  1630. /* Recovery: lock and submit the mapped buffers */
  1631. do {
  1632. if (buffer_mapped(bh) && buffer_dirty(bh) &&
  1633. !buffer_delay(bh)) {
  1634. lock_buffer(bh);
  1635. mark_buffer_async_write_endio(bh, handler);
  1636. } else {
  1637. /*
  1638. * The buffer may have been set dirty during
  1639. * attachment to a dirty page.
  1640. */
  1641. clear_buffer_dirty(bh);
  1642. }
  1643. } while ((bh = bh->b_this_page) != head);
  1644. SetPageError(page);
  1645. BUG_ON(PageWriteback(page));
  1646. mapping_set_error(page->mapping, err);
  1647. set_page_writeback(page);
  1648. do {
  1649. struct buffer_head *next = bh->b_this_page;
  1650. if (buffer_async_write(bh)) {
  1651. clear_buffer_dirty(bh);
  1652. submit_bh(write_op, bh);
  1653. nr_underway++;
  1654. }
  1655. bh = next;
  1656. } while (bh != head);
  1657. unlock_page(page);
  1658. goto done;
  1659. }
  1660. /*
  1661. * If a page has any new buffers, zero them out here, and mark them uptodate
  1662. * and dirty so they'll be written out (in order to prevent uninitialised
  1663. * block data from leaking). And clear the new bit.
  1664. */
  1665. void page_zero_new_buffers(struct page *page, unsigned from, unsigned to)
  1666. {
  1667. unsigned int block_start, block_end;
  1668. struct buffer_head *head, *bh;
  1669. BUG_ON(!PageLocked(page));
  1670. if (!page_has_buffers(page))
  1671. return;
  1672. bh = head = page_buffers(page);
  1673. block_start = 0;
  1674. do {
  1675. block_end = block_start + bh->b_size;
  1676. if (buffer_new(bh)) {
  1677. if (block_end > from && block_start < to) {
  1678. if (!PageUptodate(page)) {
  1679. unsigned start, size;
  1680. start = max(from, block_start);
  1681. size = min(to, block_end) - start;
  1682. zero_user(page, start, size);
  1683. set_buffer_uptodate(bh);
  1684. }
  1685. clear_buffer_new(bh);
  1686. mark_buffer_dirty(bh);
  1687. }
  1688. }
  1689. block_start = block_end;
  1690. bh = bh->b_this_page;
  1691. } while (bh != head);
  1692. }
  1693. EXPORT_SYMBOL(page_zero_new_buffers);
  1694. int __block_write_begin(struct page *page, loff_t pos, unsigned len,
  1695. get_block_t *get_block)
  1696. {
  1697. unsigned from = pos & (PAGE_CACHE_SIZE - 1);
  1698. unsigned to = from + len;
  1699. struct inode *inode = page->mapping->host;
  1700. unsigned block_start, block_end;
  1701. sector_t block;
  1702. int err = 0;
  1703. unsigned blocksize, bbits;
  1704. struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
  1705. BUG_ON(!PageLocked(page));
  1706. BUG_ON(from > PAGE_CACHE_SIZE);
  1707. BUG_ON(to > PAGE_CACHE_SIZE);
  1708. BUG_ON(from > to);
  1709. head = create_page_buffers(page, inode, 0);
  1710. blocksize = head->b_size;
  1711. bbits = block_size_bits(blocksize);
  1712. block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
  1713. for(bh = head, block_start = 0; bh != head || !block_start;
  1714. block++, block_start=block_end, bh = bh->b_this_page) {
  1715. block_end = block_start + blocksize;
  1716. if (block_end <= from || block_start >= to) {
  1717. if (PageUptodate(page)) {
  1718. if (!buffer_uptodate(bh))
  1719. set_buffer_uptodate(bh);
  1720. }
  1721. continue;
  1722. }
  1723. if (buffer_new(bh))
  1724. clear_buffer_new(bh);
  1725. if (!buffer_mapped(bh)) {
  1726. WARN_ON(bh->b_size != blocksize);
  1727. err = get_block(inode, block, bh, 1);
  1728. if (err)
  1729. break;
  1730. if (buffer_new(bh)) {
  1731. unmap_underlying_metadata(bh->b_bdev,
  1732. bh->b_blocknr);
  1733. if (PageUptodate(page)) {
  1734. clear_buffer_new(bh);
  1735. set_buffer_uptodate(bh);
  1736. mark_buffer_dirty(bh);
  1737. continue;
  1738. }
  1739. if (block_end > to || block_start < from)
  1740. zero_user_segments(page,
  1741. to, block_end,
  1742. block_start, from);
  1743. continue;
  1744. }
  1745. }
  1746. if (PageUptodate(page)) {
  1747. if (!buffer_uptodate(bh))
  1748. set_buffer_uptodate(bh);
  1749. continue;
  1750. }
  1751. if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
  1752. !buffer_unwritten(bh) &&
  1753. (block_start < from || block_end > to)) {
  1754. ll_rw_block(READ, 1, &bh);
  1755. *wait_bh++=bh;
  1756. }
  1757. }
  1758. /*
  1759. * If we issued read requests - let them complete.
  1760. */
  1761. while(wait_bh > wait) {
  1762. wait_on_buffer(*--wait_bh);
  1763. if (!buffer_uptodate(*wait_bh))
  1764. err = -EIO;
  1765. }
  1766. if (unlikely(err))
  1767. page_zero_new_buffers(page, from, to);
  1768. return err;
  1769. }
  1770. EXPORT_SYMBOL(__block_write_begin);
  1771. static int __block_commit_write(struct inode *inode, struct page *page,
  1772. unsigned from, unsigned to)
  1773. {
  1774. unsigned block_start, block_end;
  1775. int partial = 0;
  1776. unsigned blocksize;
  1777. struct buffer_head *bh, *head;
  1778. bh = head = page_buffers(page);
  1779. blocksize = bh->b_size;
  1780. block_start = 0;
  1781. do {
  1782. block_end = block_start + blocksize;
  1783. if (block_end <= from || block_start >= to) {
  1784. if (!buffer_uptodate(bh))
  1785. partial = 1;
  1786. } else {
  1787. set_buffer_uptodate(bh);
  1788. mark_buffer_dirty(bh);
  1789. }
  1790. clear_buffer_new(bh);
  1791. block_start = block_end;
  1792. bh = bh->b_this_page;
  1793. } while (bh != head);
  1794. /*
  1795. * If this is a partial write which happened to make all buffers
  1796. * uptodate then we can optimize away a bogus readpage() for
  1797. * the next read(). Here we 'discover' whether the page went
  1798. * uptodate as a result of this (potentially partial) write.
  1799. */
  1800. if (!partial)
  1801. SetPageUptodate(page);
  1802. return 0;
  1803. }
  1804. /*
  1805. * block_write_begin takes care of the basic task of block allocation and
  1806. * bringing partial write blocks uptodate first.
  1807. *
  1808. * The filesystem needs to handle block truncation upon failure.
  1809. */
  1810. int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
  1811. unsigned flags, struct page **pagep, get_block_t *get_block)
  1812. {
  1813. pgoff_t index = pos >> PAGE_CACHE_SHIFT;
  1814. struct page *page;
  1815. int status;
  1816. page = grab_cache_page_write_begin(mapping, index, flags);
  1817. if (!page)
  1818. return -ENOMEM;
  1819. status = __block_write_begin(page, pos, len, get_block);
  1820. if (unlikely(status)) {
  1821. unlock_page(page);
  1822. page_cache_release(page);
  1823. page = NULL;
  1824. }
  1825. *pagep = page;
  1826. return status;
  1827. }
  1828. EXPORT_SYMBOL(block_write_begin);
  1829. int block_write_end(struct file *file, struct address_space *mapping,
  1830. loff_t pos, unsigned len, unsigned copied,
  1831. struct page *page, void *fsdata)
  1832. {
  1833. struct inode *inode = mapping->host;
  1834. unsigned start;
  1835. start = pos & (PAGE_CACHE_SIZE - 1);
  1836. if (unlikely(copied < len)) {
  1837. /*
  1838. * The buffers that were written will now be uptodate, so we
  1839. * don't have to worry about a readpage reading them and
  1840. * overwriting a partial write. However if we have encountered
  1841. * a short write and only partially written into a buffer, it
  1842. * will not be marked uptodate, so a readpage might come in and
  1843. * destroy our partial write.
  1844. *
  1845. * Do the simplest thing, and just treat any short write to a
  1846. * non uptodate page as a zero-length write, and force the
  1847. * caller to redo the whole thing.
  1848. */
  1849. if (!PageUptodate(page))
  1850. copied = 0;
  1851. page_zero_new_buffers(page, start+copied, start+len);
  1852. }
  1853. flush_dcache_page(page);
  1854. /* This could be a short (even 0-length) commit */
  1855. __block_commit_write(inode, page, start, start+copied);
  1856. return copied;
  1857. }
  1858. EXPORT_SYMBOL(block_write_end);
  1859. int generic_write_end(struct file *file, struct address_space *mapping,
  1860. loff_t pos, unsigned len, unsigned copied,
  1861. struct page *page, void *fsdata)
  1862. {
  1863. struct inode *inode = mapping->host;
  1864. int i_size_changed = 0;
  1865. copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
  1866. /*
  1867. * No need to use i_size_read() here, the i_size
  1868. * cannot change under us because we hold i_mutex.
  1869. *
  1870. * But it's important to update i_size while still holding page lock:
  1871. * page writeout could otherwise come in and zero beyond i_size.
  1872. */
  1873. if (pos+copied > inode->i_size) {
  1874. i_size_write(inode, pos+copied);
  1875. i_size_changed = 1;
  1876. }
  1877. unlock_page(page);
  1878. page_cache_release(page);
  1879. /*
  1880. * Don't mark the inode dirty under page lock. First, it unnecessarily
  1881. * makes the holding time of page lock longer. Second, it forces lock
  1882. * ordering of page lock and transaction start for journaling
  1883. * filesystems.
  1884. */
  1885. if (i_size_changed)
  1886. mark_inode_dirty(inode);
  1887. return copied;
  1888. }
  1889. EXPORT_SYMBOL(generic_write_end);
  1890. /*
  1891. * block_is_partially_uptodate checks whether buffers within a page are
  1892. * uptodate or not.
  1893. *
  1894. * Returns true if all buffers which correspond to a file portion
  1895. * we want to read are uptodate.
  1896. */
  1897. int block_is_partially_uptodate(struct page *page, read_descriptor_t *desc,
  1898. unsigned long from)
  1899. {
  1900. unsigned block_start, block_end, blocksize;
  1901. unsigned to;
  1902. struct buffer_head *bh, *head;
  1903. int ret = 1;
  1904. if (!page_has_buffers(page))
  1905. return 0;
  1906. head = page_buffers(page);
  1907. blocksize = head->b_size;
  1908. to = min_t(unsigned, PAGE_CACHE_SIZE - from, desc->count);
  1909. to = from + to;
  1910. if (from < blocksize && to > PAGE_CACHE_SIZE - blocksize)
  1911. return 0;
  1912. bh = head;
  1913. block_start = 0;
  1914. do {
  1915. block_end = block_start + blocksize;
  1916. if (block_end > from && block_start < to) {
  1917. if (!buffer_uptodate(bh)) {
  1918. ret = 0;
  1919. break;
  1920. }
  1921. if (block_end >= to)
  1922. break;
  1923. }
  1924. block_start = block_end;
  1925. bh = bh->b_this_page;
  1926. } while (bh != head);
  1927. return ret;
  1928. }
  1929. EXPORT_SYMBOL(block_is_partially_uptodate);
  1930. /*
  1931. * Generic "read page" function for block devices that have the normal
  1932. * get_block functionality. This is most of the block device filesystems.
  1933. * Reads the page asynchronously --- the unlock_buffer() and
  1934. * set/clear_buffer_uptodate() functions propagate buffer state into the
  1935. * page struct once IO has completed.
  1936. */
  1937. int block_read_full_page(struct page *page, get_block_t *get_block)
  1938. {
  1939. struct inode *inode = page->mapping->host;
  1940. sector_t iblock, lblock;
  1941. struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
  1942. unsigned int blocksize, bbits;
  1943. int nr, i;
  1944. int fully_mapped = 1;
  1945. head = create_page_buffers(page, inode, 0);
  1946. blocksize = head->b_size;
  1947. bbits = block_size_bits(blocksize);
  1948. iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
  1949. lblock = (i_size_read(inode)+blocksize-1) >> bbits;
  1950. bh = head;
  1951. nr = 0;
  1952. i = 0;
  1953. do {
  1954. if (buffer_uptodate(bh))
  1955. continue;
  1956. if (!buffer_mapped(bh)) {
  1957. int err = 0;
  1958. fully_mapped = 0;
  1959. if (iblock < lblock) {
  1960. WARN_ON(bh->b_size != blocksize);
  1961. err = get_block(inode, iblock, bh, 0);
  1962. if (err)
  1963. SetPageError(page);
  1964. }
  1965. if (!buffer_mapped(bh)) {
  1966. zero_user(page, i * blocksize, blocksize);
  1967. if (!err)
  1968. set_buffer_uptodate(bh);
  1969. continue;
  1970. }
  1971. /*
  1972. * get_block() might have updated the buffer
  1973. * synchronously
  1974. */
  1975. if (buffer_uptodate(bh))
  1976. continue;
  1977. }
  1978. arr[nr++] = bh;
  1979. } while (i++, iblock++, (bh = bh->b_this_page) != head);
  1980. if (fully_mapped)
  1981. SetPageMappedToDisk(page);
  1982. if (!nr) {
  1983. /*
  1984. * All buffers are uptodate - we can set the page uptodate
  1985. * as well. But not if get_block() returned an error.
  1986. */
  1987. if (!PageError(page))
  1988. SetPageUptodate(page);
  1989. unlock_page(page);
  1990. return 0;
  1991. }
  1992. /* Stage two: lock the buffers */
  1993. for (i = 0; i < nr; i++) {
  1994. bh = arr[i];
  1995. lock_buffer(bh);
  1996. mark_buffer_async_read(bh);
  1997. }
  1998. /*
  1999. * Stage 3: start the IO. Check for uptodateness
  2000. * inside the buffer lock in case another process reading
  2001. * the underlying blockdev brought it uptodate (the sct fix).
  2002. */
  2003. for (i = 0; i < nr; i++) {
  2004. bh = arr[i];
  2005. if (buffer_uptodate(bh))
  2006. end_buffer_async_read(bh, 1);
  2007. else
  2008. submit_bh(READ, bh);
  2009. }
  2010. return 0;
  2011. }
  2012. EXPORT_SYMBOL(block_read_full_page);
  2013. /* utility function for filesystems that need to do work on expanding
  2014. * truncates. Uses filesystem pagecache writes to allow the filesystem to
  2015. * deal with the hole.
  2016. */
  2017. int generic_cont_expand_simple(struct inode *inode, loff_t size)
  2018. {
  2019. struct address_space *mapping = inode->i_mapping;
  2020. struct page *page;
  2021. void *fsdata;
  2022. int err;
  2023. err = inode_newsize_ok(inode, size);
  2024. if (err)
  2025. goto out;
  2026. err = pagecache_write_begin(NULL, mapping, size, 0,
  2027. AOP_FLAG_UNINTERRUPTIBLE|AOP_FLAG_CONT_EXPAND,
  2028. &page, &fsdata);
  2029. if (err)
  2030. goto out;
  2031. err = pagecache_write_end(NULL, mapping, size, 0, 0, page, fsdata);
  2032. BUG_ON(err > 0);
  2033. out:
  2034. return err;
  2035. }
  2036. EXPORT_SYMBOL(generic_cont_expand_simple);
  2037. static int cont_expand_zero(struct file *file, struct address_space *mapping,
  2038. loff_t pos, loff_t *bytes)
  2039. {
  2040. struct inode *inode = mapping->host;
  2041. unsigned blocksize = 1 << inode->i_blkbits;
  2042. struct page *page;
  2043. void *fsdata;
  2044. pgoff_t index, curidx;
  2045. loff_t curpos;
  2046. unsigned zerofrom, offset, len;
  2047. int err = 0;
  2048. index = pos >> PAGE_CACHE_SHIFT;
  2049. offset = pos & ~PAGE_CACHE_MASK;
  2050. while (index > (curidx = (curpos = *bytes)>>PAGE_CACHE_SHIFT)) {
  2051. zerofrom = curpos & ~PAGE_CACHE_MASK;
  2052. if (zerofrom & (blocksize-1)) {
  2053. *bytes |= (blocksize-1);
  2054. (*bytes)++;
  2055. }
  2056. len = PAGE_CACHE_SIZE - zerofrom;
  2057. err = pagecache_write_begin(file, mapping, curpos, len,
  2058. AOP_FLAG_UNINTERRUPTIBLE,
  2059. &page, &fsdata);
  2060. if (err)
  2061. goto out;
  2062. zero_user(page, zerofrom, len);
  2063. err = pagecache_write_end(file, mapping, curpos, len, len,
  2064. page, fsdata);
  2065. if (err < 0)
  2066. goto out;
  2067. BUG_ON(err != len);
  2068. err = 0;
  2069. balance_dirty_pages_ratelimited(mapping);
  2070. }
  2071. /* page covers the boundary, find the boundary offset */
  2072. if (index == curidx) {
  2073. zerofrom = curpos & ~PAGE_CACHE_MASK;
  2074. /* if we will expand the thing last block will be filled */
  2075. if (offset <= zerofrom) {
  2076. goto out;
  2077. }
  2078. if (zerofrom & (blocksize-1)) {
  2079. *bytes |= (blocksize-1);
  2080. (*bytes)++;
  2081. }
  2082. len = offset - zerofrom;
  2083. err = pagecache_write_begin(file, mapping, curpos, len,
  2084. AOP_FLAG_UNINTERRUPTIBLE,
  2085. &page, &fsdata);
  2086. if (err)
  2087. goto out;
  2088. zero_user(page, zerofrom, len);
  2089. err = pagecache_write_end(file, mapping, curpos, len, len,
  2090. page, fsdata);
  2091. if (err < 0)
  2092. goto out;
  2093. BUG_ON(err != len);
  2094. err = 0;
  2095. }
  2096. out:
  2097. return err;
  2098. }
  2099. /*
  2100. * For moronic filesystems that do not allow holes in file.
  2101. * We may have to extend the file.
  2102. */
  2103. int cont_write_begin(struct file *file, struct address_space *mapping,
  2104. loff_t pos, unsigned len, unsigned flags,
  2105. struct page **pagep, void **fsdata,
  2106. get_block_t *get_block, loff_t *bytes)
  2107. {
  2108. struct inode *inode = mapping->host;
  2109. unsigned blocksize = 1 << inode->i_blkbits;
  2110. unsigned zerofrom;
  2111. int err;
  2112. err = cont_expand_zero(file, mapping, pos, bytes);
  2113. if (err)
  2114. return err;
  2115. zerofrom = *bytes & ~PAGE_CACHE_MASK;
  2116. if (pos+len > *bytes && zerofrom & (blocksize-1)) {
  2117. *bytes |= (blocksize-1);
  2118. (*bytes)++;
  2119. }
  2120. return block_write_begin(mapping, pos, len, flags, pagep, get_block);
  2121. }
  2122. EXPORT_SYMBOL(cont_write_begin);
  2123. int block_commit_write(struct page *page, unsigned from, unsigned to)
  2124. {
  2125. struct inode *inode = page->mapping->host;
  2126. __block_commit_write(inode,page,from,to);
  2127. return 0;
  2128. }
  2129. EXPORT_SYMBOL(block_commit_write);
  2130. /*
  2131. * block_page_mkwrite() is not allowed to change the file size as it gets
  2132. * called from a page fault handler when a page is first dirtied. Hence we must
  2133. * be careful to check for EOF conditions here. We set the page up correctly
  2134. * for a written page which means we get ENOSPC checking when writing into
  2135. * holes and correct delalloc and unwritten extent mapping on filesystems that
  2136. * support these features.
  2137. *
  2138. * We are not allowed to take the i_mutex here so we have to play games to
  2139. * protect against truncate races as the page could now be beyond EOF. Because
  2140. * truncate writes the inode size before removing pages, once we have the
  2141. * page lock we can determine safely if the page is beyond EOF. If it is not
  2142. * beyond EOF, then the page is guaranteed safe against truncation until we
  2143. * unlock the page.
  2144. *
  2145. * Direct callers of this function should protect against filesystem freezing
  2146. * using sb_start_write() - sb_end_write() functions.
  2147. */
  2148. int __block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
  2149. get_block_t get_block)
  2150. {
  2151. struct page *page = vmf->page;
  2152. struct inode *inode = file_inode(vma->vm_file);
  2153. unsigned long end;
  2154. loff_t size;
  2155. int ret;
  2156. lock_page(page);
  2157. size = i_size_read(inode);
  2158. if ((page->mapping != inode->i_mapping) ||
  2159. (page_offset(page) > size)) {
  2160. /* We overload EFAULT to mean page got truncated */
  2161. ret = -EFAULT;
  2162. goto out_unlock;
  2163. }
  2164. /* page is wholly or partially inside EOF */
  2165. if (((page->index + 1) << PAGE_CACHE_SHIFT) > size)
  2166. end = size & ~PAGE_CACHE_MASK;
  2167. else
  2168. end = PAGE_CACHE_SIZE;
  2169. ret = __block_write_begin(page, 0, end, get_block);
  2170. if (!ret)
  2171. ret = block_commit_write(page, 0, end);
  2172. if (unlikely(ret < 0))
  2173. goto out_unlock;
  2174. set_page_dirty(page);
  2175. wait_for_stable_page(page);
  2176. return 0;
  2177. out_unlock:
  2178. unlock_page(page);
  2179. return ret;
  2180. }
  2181. EXPORT_SYMBOL(__block_page_mkwrite);
  2182. int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
  2183. get_block_t get_block)
  2184. {
  2185. int ret;
  2186. struct super_block *sb = file_inode(vma->vm_file)->i_sb;
  2187. sb_start_pagefault(sb);
  2188. /*
  2189. * Update file times before taking page lock. We may end up failing the
  2190. * fault so this update may be superfluous but who really cares...
  2191. */
  2192. file_update_time(vma->vm_file);
  2193. ret = __block_page_mkwrite(vma, vmf, get_block);
  2194. sb_end_pagefault(sb);
  2195. return block_page_mkwrite_return(ret);
  2196. }
  2197. EXPORT_SYMBOL(block_page_mkwrite);
  2198. /*
  2199. * nobh_write_begin()'s prereads are special: the buffer_heads are freed
  2200. * immediately, while under the page lock. So it needs a special end_io
  2201. * handler which does not touch the bh after unlocking it.
  2202. */
  2203. static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
  2204. {
  2205. __end_buffer_read_notouch(bh, uptodate);
  2206. }
  2207. /*
  2208. * Attach the singly-linked list of buffers created by nobh_write_begin, to
  2209. * the page (converting it to circular linked list and taking care of page
  2210. * dirty races).
  2211. */
  2212. static void attach_nobh_buffers(struct page *page, struct buffer_head *head)
  2213. {
  2214. struct buffer_head *bh;
  2215. BUG_ON(!PageLocked(page));
  2216. spin_lock(&page->mapping->private_lock);
  2217. bh = head;
  2218. do {
  2219. if (PageDirty(page))
  2220. set_buffer_dirty(bh);
  2221. if (!bh->b_this_page)
  2222. bh->b_this_page = head;
  2223. bh = bh->b_this_page;
  2224. } while (bh != head);
  2225. attach_page_buffers(page, head);
  2226. spin_unlock(&page->mapping->private_lock);
  2227. }
  2228. /*
  2229. * On entry, the page is fully not uptodate.
  2230. * On exit the page is fully uptodate in the areas outside (from,to)
  2231. * The filesystem needs to handle block truncation upon failure.
  2232. */
  2233. int nobh_write_begin(struct address_space *mapping,
  2234. loff_t pos, unsigned len, unsigned flags,
  2235. struct page **pagep, void **fsdata,
  2236. get_block_t *get_block)
  2237. {
  2238. struct inode *inode = mapping->host;
  2239. const unsigned blkbits = inode->i_blkbits;
  2240. const unsigned blocksize = 1 << blkbits;
  2241. struct buffer_head *head, *bh;
  2242. struct page *page;
  2243. pgoff_t index;
  2244. unsigned from, to;
  2245. unsigned block_in_page;
  2246. unsigned block_start, block_end;
  2247. sector_t block_in_file;
  2248. int nr_reads = 0;
  2249. int ret = 0;
  2250. int is_mapped_to_disk = 1;
  2251. index = pos >> PAGE_CACHE_SHIFT;
  2252. from = pos & (PAGE_CACHE_SIZE - 1);
  2253. to = from + len;
  2254. page = grab_cache_page_write_begin(mapping, index, flags);
  2255. if (!page)
  2256. return -ENOMEM;
  2257. *pagep = page;
  2258. *fsdata = NULL;
  2259. if (page_has_buffers(page)) {
  2260. ret = __block_write_begin(page, pos, len, get_block);
  2261. if (unlikely(ret))
  2262. goto out_release;
  2263. return ret;
  2264. }
  2265. if (PageMappedToDisk(page))
  2266. return 0;
  2267. /*
  2268. * Allocate buffers so that we can keep track of state, and potentially
  2269. * attach them to the page if an error occurs. In the common case of
  2270. * no error, they will just be freed again without ever being attached
  2271. * to the page (which is all OK, because we're under the page lock).
  2272. *
  2273. * Be careful: the buffer linked list is a NULL terminated one, rather
  2274. * than the circular one we're used to.
  2275. */
  2276. head = alloc_page_buffers(page, blocksize, 0);
  2277. if (!head) {
  2278. ret = -ENOMEM;
  2279. goto out_release;
  2280. }
  2281. block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
  2282. /*
  2283. * We loop across all blocks in the page, whether or not they are
  2284. * part of the affected region. This is so we can discover if the
  2285. * page is fully mapped-to-disk.
  2286. */
  2287. for (block_start = 0, block_in_page = 0, bh = head;
  2288. block_start < PAGE_CACHE_SIZE;
  2289. block_in_page++, block_start += blocksize, bh = bh->b_this_page) {
  2290. int create;
  2291. block_end = block_start + blocksize;
  2292. bh->b_state = 0;
  2293. create = 1;
  2294. if (block_start >= to)
  2295. create = 0;
  2296. ret = get_block(inode, block_in_file + block_in_page,
  2297. bh, create);
  2298. if (ret)
  2299. goto failed;
  2300. if (!buffer_mapped(bh))
  2301. is_mapped_to_disk = 0;
  2302. if (buffer_new(bh))
  2303. unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr);
  2304. if (PageUptodate(page)) {
  2305. set_buffer_uptodate(bh);
  2306. continue;
  2307. }
  2308. if (buffer_new(bh) || !buffer_mapped(bh)) {
  2309. zero_user_segments(page, block_start, from,
  2310. to, block_end);
  2311. continue;
  2312. }
  2313. if (buffer_uptodate(bh))
  2314. continue; /* reiserfs does this */
  2315. if (block_start < from || block_end > to) {
  2316. lock_buffer(bh);
  2317. bh->b_end_io = end_buffer_read_nobh;
  2318. submit_bh(READ, bh);
  2319. nr_reads++;
  2320. }
  2321. }
  2322. if (nr_reads) {
  2323. /*
  2324. * The page is locked, so these buffers are protected from
  2325. * any VM or truncate activity. Hence we don't need to care
  2326. * for the buffer_head refcounts.
  2327. */
  2328. for (bh = head; bh; bh = bh->b_this_page) {
  2329. wait_on_buffer(bh);
  2330. if (!buffer_uptodate(bh))
  2331. ret = -EIO;
  2332. }
  2333. if (ret)
  2334. goto failed;
  2335. }
  2336. if (is_mapped_to_disk)
  2337. SetPageMappedToDisk(page);
  2338. *fsdata = head; /* to be released by nobh_write_end */
  2339. return 0;
  2340. failed:
  2341. BUG_ON(!ret);
  2342. /*
  2343. * Error recovery is a bit difficult. We need to zero out blocks that
  2344. * were newly allocated, and dirty them to ensure they get written out.
  2345. * Buffers need to be attached to the page at this point, otherwise
  2346. * the handling of potential IO errors during writeout would be hard
  2347. * (could try doing synchronous writeout, but what if that fails too?)
  2348. */
  2349. attach_nobh_buffers(page, head);
  2350. page_zero_new_buffers(page, from, to);
  2351. out_release:
  2352. unlock_page(page);
  2353. page_cache_release(page);
  2354. *pagep = NULL;
  2355. return ret;
  2356. }
  2357. EXPORT_SYMBOL(nobh_write_begin);
  2358. int nobh_write_end(struct file *file, struct address_space *mapping,
  2359. loff_t pos, unsigned len, unsigned copied,
  2360. struct page *page, void *fsdata)
  2361. {
  2362. struct inode *inode = page->mapping->host;
  2363. struct buffer_head *head = fsdata;
  2364. struct buffer_head *bh;
  2365. BUG_ON(fsdata != NULL && page_has_buffers(page));
  2366. if (unlikely(copied < len) && head)
  2367. attach_nobh_buffers(page, head);
  2368. if (page_has_buffers(page))
  2369. return generic_write_end(file, mapping, pos, len,
  2370. copied, page, fsdata);
  2371. SetPageUptodate(page);
  2372. set_page_dirty(page);
  2373. if (pos+copied > inode->i_size) {
  2374. i_size_write(inode, pos+copied);
  2375. mark_inode_dirty(inode);
  2376. }
  2377. unlock_page(page);
  2378. page_cache_release(page);
  2379. while (head) {
  2380. bh = head;
  2381. head = head->b_this_page;
  2382. free_buffer_head(bh);
  2383. }
  2384. return copied;
  2385. }
  2386. EXPORT_SYMBOL(nobh_write_end);
  2387. /*
  2388. * nobh_writepage() - based on block_full_write_page() except
  2389. * that it tries to operate without attaching bufferheads to
  2390. * the page.
  2391. */
  2392. int nobh_writepage(struct page *page, get_block_t *get_block,
  2393. struct writeback_control *wbc)
  2394. {
  2395. struct inode * const inode = page->mapping->host;
  2396. loff_t i_size = i_size_read(inode);
  2397. const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
  2398. unsigned offset;
  2399. int ret;
  2400. /* Is the page fully inside i_size? */
  2401. if (page->index < end_index)
  2402. goto out;
  2403. /* Is the page fully outside i_size? (truncate in progress) */
  2404. offset = i_size & (PAGE_CACHE_SIZE-1);
  2405. if (page->index >= end_index+1 || !offset) {
  2406. /*
  2407. * The page may have dirty, unmapped buffers. For example,
  2408. * they may have been added in ext3_writepage(). Make them
  2409. * freeable here, so the page does not leak.
  2410. */
  2411. #if 0
  2412. /* Not really sure about this - do we need this ? */
  2413. if (page->mapping->a_ops->invalidatepage)
  2414. page->mapping->a_ops->invalidatepage(page, offset);
  2415. #endif
  2416. unlock_page(page);
  2417. return 0; /* don't care */
  2418. }
  2419. /*
  2420. * The page straddles i_size. It must be zeroed out on each and every
  2421. * writepage invocation because it may be mmapped. "A file is mapped
  2422. * in multiples of the page size. For a file that is not a multiple of
  2423. * the page size, the remaining memory is zeroed when mapped, and
  2424. * writes to that region are not written out to the file."
  2425. */
  2426. zero_user_segment(page, offset, PAGE_CACHE_SIZE);
  2427. out:
  2428. ret = mpage_writepage(page, get_block, wbc);
  2429. if (ret == -EAGAIN)
  2430. ret = __block_write_full_page(inode, page, get_block, wbc,
  2431. end_buffer_async_write);
  2432. return ret;
  2433. }
  2434. EXPORT_SYMBOL(nobh_writepage);
  2435. int nobh_truncate_page(struct address_space *mapping,
  2436. loff_t from, get_block_t *get_block)
  2437. {
  2438. pgoff_t index = from >> PAGE_CACHE_SHIFT;
  2439. unsigned offset = from & (PAGE_CACHE_SIZE-1);
  2440. unsigned blocksize;
  2441. sector_t iblock;
  2442. unsigned length, pos;
  2443. struct inode *inode = mapping->host;
  2444. struct page *page;
  2445. struct buffer_head map_bh;
  2446. int err;
  2447. blocksize = 1 << inode->i_blkbits;
  2448. length = offset & (blocksize - 1);
  2449. /* Block boundary? Nothing to do */
  2450. if (!length)
  2451. return 0;
  2452. length = blocksize - length;
  2453. iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
  2454. page = grab_cache_page(mapping, index);
  2455. err = -ENOMEM;
  2456. if (!page)
  2457. goto out;
  2458. if (page_has_buffers(page)) {
  2459. has_buffers:
  2460. unlock_page(page);
  2461. page_cache_release(page);
  2462. return block_truncate_page(mapping, from, get_block);
  2463. }
  2464. /* Find the buffer that contains "offset" */
  2465. pos = blocksize;
  2466. while (offset >= pos) {
  2467. iblock++;
  2468. pos += blocksize;
  2469. }
  2470. map_bh.b_size = blocksize;
  2471. map_bh.b_state = 0;
  2472. err = get_block(inode, iblock, &map_bh, 0);
  2473. if (err)
  2474. goto unlock;
  2475. /* unmapped? It's a hole - nothing to do */
  2476. if (!buffer_mapped(&map_bh))
  2477. goto unlock;
  2478. /* Ok, it's mapped. Make sure it's up-to-date */
  2479. if (!PageUptodate(page)) {
  2480. err = mapping->a_ops->readpage(NULL, page);
  2481. if (err) {
  2482. page_cache_release(page);
  2483. goto out;
  2484. }
  2485. lock_page(page);
  2486. if (!PageUptodate(page)) {
  2487. err = -EIO;
  2488. goto unlock;
  2489. }
  2490. if (page_has_buffers(page))
  2491. goto has_buffers;
  2492. }
  2493. zero_user(page, offset, length);
  2494. set_page_dirty(page);
  2495. err = 0;
  2496. unlock:
  2497. unlock_page(page);
  2498. page_cache_release(page);
  2499. out:
  2500. return err;
  2501. }
  2502. EXPORT_SYMBOL(nobh_truncate_page);
  2503. int block_truncate_page(struct address_space *mapping,
  2504. loff_t from, get_block_t *get_block)
  2505. {
  2506. pgoff_t index = from >> PAGE_CACHE_SHIFT;
  2507. unsigned offset = from & (PAGE_CACHE_SIZE-1);
  2508. unsigned blocksize;
  2509. sector_t iblock;
  2510. unsigned length, pos;
  2511. struct inode *inode = mapping->host;
  2512. struct page *page;
  2513. struct buffer_head *bh;
  2514. int err;
  2515. blocksize = 1 << inode->i_blkbits;
  2516. length = offset & (blocksize - 1);
  2517. /* Block boundary? Nothing to do */
  2518. if (!length)
  2519. return 0;
  2520. length = blocksize - length;
  2521. iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
  2522. page = grab_cache_page(mapping, index);
  2523. err = -ENOMEM;
  2524. if (!page)
  2525. goto out;
  2526. if (!page_has_buffers(page))
  2527. create_empty_buffers(page, blocksize, 0);
  2528. /* Find the buffer that contains "offset" */
  2529. bh = page_buffers(page);
  2530. pos = blocksize;
  2531. while (offset >= pos) {
  2532. bh = bh->b_this_page;
  2533. iblock++;
  2534. pos += blocksize;
  2535. }
  2536. err = 0;
  2537. if (!buffer_mapped(bh)) {
  2538. WARN_ON(bh->b_size != blocksize);
  2539. err = get_block(inode, iblock, bh, 0);
  2540. if (err)
  2541. goto unlock;
  2542. /* unmapped? It's a hole - nothing to do */
  2543. if (!buffer_mapped(bh))
  2544. goto unlock;
  2545. }
  2546. /* Ok, it's mapped. Make sure it's up-to-date */
  2547. if (PageUptodate(page))
  2548. set_buffer_uptodate(bh);
  2549. if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
  2550. err = -EIO;
  2551. ll_rw_block(READ, 1, &bh);
  2552. wait_on_buffer(bh);
  2553. /* Uhhuh. Read error. Complain and punt. */
  2554. if (!buffer_uptodate(bh))
  2555. goto unlock;
  2556. }
  2557. zero_user(page, offset, length);
  2558. mark_buffer_dirty(bh);
  2559. err = 0;
  2560. unlock:
  2561. unlock_page(page);
  2562. page_cache_release(page);
  2563. out:
  2564. return err;
  2565. }
  2566. EXPORT_SYMBOL(block_truncate_page);
  2567. /*
  2568. * The generic ->writepage function for buffer-backed address_spaces
  2569. * this form passes in the end_io handler used to finish the IO.
  2570. */
  2571. int block_write_full_page_endio(struct page *page, get_block_t *get_block,
  2572. struct writeback_control *wbc, bh_end_io_t *handler)
  2573. {
  2574. struct inode * const inode = page->mapping->host;
  2575. loff_t i_size = i_size_read(inode);
  2576. const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
  2577. unsigned offset;
  2578. /* Is the page fully inside i_size? */
  2579. if (page->index < end_index)
  2580. return __block_write_full_page(inode, page, get_block, wbc,
  2581. handler);
  2582. /* Is the page fully outside i_size? (truncate in progress) */
  2583. offset = i_size & (PAGE_CACHE_SIZE-1);
  2584. if (page->index >= end_index+1 || !offset) {
  2585. /*
  2586. * The page may have dirty, unmapped buffers. For example,
  2587. * they may have been added in ext3_writepage(). Make them
  2588. * freeable here, so the page does not leak.
  2589. */
  2590. do_invalidatepage(page, 0, PAGE_CACHE_SIZE);
  2591. unlock_page(page);
  2592. return 0; /* don't care */
  2593. }
  2594. /*
  2595. * The page straddles i_size. It must be zeroed out on each and every
  2596. * writepage invocation because it may be mmapped. "A file is mapped
  2597. * in multiples of the page size. For a file that is not a multiple of
  2598. * the page size, the remaining memory is zeroed when mapped, and
  2599. * writes to that region are not written out to the file."
  2600. */
  2601. zero_user_segment(page, offset, PAGE_CACHE_SIZE);
  2602. return __block_write_full_page(inode, page, get_block, wbc, handler);
  2603. }
  2604. EXPORT_SYMBOL(block_write_full_page_endio);
  2605. /*
  2606. * The generic ->writepage function for buffer-backed address_spaces
  2607. */
  2608. int block_write_full_page(struct page *page, get_block_t *get_block,
  2609. struct writeback_control *wbc)
  2610. {
  2611. return block_write_full_page_endio(page, get_block, wbc,
  2612. end_buffer_async_write);
  2613. }
  2614. EXPORT_SYMBOL(block_write_full_page);
  2615. sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
  2616. get_block_t *get_block)
  2617. {
  2618. struct buffer_head tmp;
  2619. struct inode *inode = mapping->host;
  2620. tmp.b_state = 0;
  2621. tmp.b_blocknr = 0;
  2622. tmp.b_size = 1 << inode->i_blkbits;
  2623. get_block(inode, block, &tmp, 0);
  2624. return tmp.b_blocknr;
  2625. }
  2626. EXPORT_SYMBOL(generic_block_bmap);
  2627. static void end_bio_bh_io_sync(struct bio *bio, int err)
  2628. {
  2629. struct buffer_head *bh = bio->bi_private;
  2630. if (err == -EOPNOTSUPP) {
  2631. set_bit(BIO_EOPNOTSUPP, &bio->bi_flags);
  2632. }
  2633. if (unlikely (test_bit(BIO_QUIET,&bio->bi_flags)))
  2634. set_bit(BH_Quiet, &bh->b_state);
  2635. bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags));
  2636. bio_put(bio);
  2637. }
  2638. /*
  2639. * This allows us to do IO even on the odd last sectors
  2640. * of a device, even if the bh block size is some multiple
  2641. * of the physical sector size.
  2642. *
  2643. * We'll just truncate the bio to the size of the device,
  2644. * and clear the end of the buffer head manually.
  2645. *
  2646. * Truly out-of-range accesses will turn into actual IO
  2647. * errors, this only handles the "we need to be able to
  2648. * do IO at the final sector" case.
  2649. */
  2650. static void guard_bh_eod(int rw, struct bio *bio, struct buffer_head *bh)
  2651. {
  2652. sector_t maxsector;
  2653. unsigned bytes;
  2654. maxsector = i_size_read(bio->bi_bdev->bd_inode) >> 9;
  2655. if (!maxsector)
  2656. return;
  2657. /*
  2658. * If the *whole* IO is past the end of the device,
  2659. * let it through, and the IO layer will turn it into
  2660. * an EIO.
  2661. */
  2662. if (unlikely(bio->bi_sector >= maxsector))
  2663. return;
  2664. maxsector -= bio->bi_sector;
  2665. bytes = bio->bi_size;
  2666. if (likely((bytes >> 9) <= maxsector))
  2667. return;
  2668. /* Uhhuh. We've got a bh that straddles the device size! */
  2669. bytes = maxsector << 9;
  2670. /* Truncate the bio.. */
  2671. bio->bi_size = bytes;
  2672. bio->bi_io_vec[0].bv_len = bytes;
  2673. /* ..and clear the end of the buffer for reads */
  2674. if ((rw & RW_MASK) == READ) {
  2675. void *kaddr = kmap_atomic(bh->b_page);
  2676. memset(kaddr + bh_offset(bh) + bytes, 0, bh->b_size - bytes);
  2677. kunmap_atomic(kaddr);
  2678. flush_dcache_page(bh->b_page);
  2679. }
  2680. }
  2681. int _submit_bh(int rw, struct buffer_head *bh, unsigned long bio_flags)
  2682. {
  2683. struct bio *bio;
  2684. int ret = 0;
  2685. BUG_ON(!buffer_locked(bh));
  2686. BUG_ON(!buffer_mapped(bh));
  2687. BUG_ON(!bh->b_end_io);
  2688. BUG_ON(buffer_delay(bh));
  2689. BUG_ON(buffer_unwritten(bh));
  2690. /*
  2691. * Only clear out a write error when rewriting
  2692. */
  2693. if (test_set_buffer_req(bh) && (rw & WRITE))
  2694. clear_buffer_write_io_error(bh);
  2695. /*
  2696. * from here on down, it's all bio -- do the initial mapping,
  2697. * submit_bio -> generic_make_request may further map this bio around
  2698. */
  2699. bio = bio_alloc(GFP_NOIO, 1);
  2700. bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
  2701. bio->bi_bdev = bh->b_bdev;
  2702. bio->bi_io_vec[0].bv_page = bh->b_page;
  2703. bio->bi_io_vec[0].bv_len = bh->b_size;
  2704. bio->bi_io_vec[0].bv_offset = bh_offset(bh);
  2705. bio->bi_vcnt = 1;
  2706. bio->bi_size = bh->b_size;
  2707. bio->bi_end_io = end_bio_bh_io_sync;
  2708. bio->bi_private = bh;
  2709. bio->bi_flags |= bio_flags;
  2710. /* Take care of bh's that straddle the end of the device */
  2711. guard_bh_eod(rw, bio, bh);
  2712. if (buffer_meta(bh))
  2713. rw |= REQ_META;
  2714. if (buffer_prio(bh))
  2715. rw |= REQ_PRIO;
  2716. bio_get(bio);
  2717. submit_bio(rw, bio);
  2718. if (bio_flagged(bio, BIO_EOPNOTSUPP))
  2719. ret = -EOPNOTSUPP;
  2720. bio_put(bio);
  2721. return ret;
  2722. }
  2723. EXPORT_SYMBOL_GPL(_submit_bh);
  2724. int submit_bh(int rw, struct buffer_head *bh)
  2725. {
  2726. return _submit_bh(rw, bh, 0);
  2727. }
  2728. EXPORT_SYMBOL(submit_bh);
  2729. /**
  2730. * ll_rw_block: low-level access to block devices (DEPRECATED)
  2731. * @rw: whether to %READ or %WRITE or maybe %READA (readahead)
  2732. * @nr: number of &struct buffer_heads in the array
  2733. * @bhs: array of pointers to &struct buffer_head
  2734. *
  2735. * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
  2736. * requests an I/O operation on them, either a %READ or a %WRITE. The third
  2737. * %READA option is described in the documentation for generic_make_request()
  2738. * which ll_rw_block() calls.
  2739. *
  2740. * This function drops any buffer that it cannot get a lock on (with the
  2741. * BH_Lock state bit), any buffer that appears to be clean when doing a write
  2742. * request, and any buffer that appears to be up-to-date when doing read
  2743. * request. Further it marks as clean buffers that are processed for
  2744. * writing (the buffer cache won't assume that they are actually clean
  2745. * until the buffer gets unlocked).
  2746. *
  2747. * ll_rw_block sets b_end_io to simple completion handler that marks
  2748. * the buffer up-to-date (if approriate), unlocks the buffer and wakes
  2749. * any waiters.
  2750. *
  2751. * All of the buffers must be for the same device, and must also be a
  2752. * multiple of the current approved size for the device.
  2753. */
  2754. void ll_rw_block(int rw, int nr, struct buffer_head *bhs[])
  2755. {
  2756. int i;
  2757. for (i = 0; i < nr; i++) {
  2758. struct buffer_head *bh = bhs[i];
  2759. if (!trylock_buffer(bh))
  2760. continue;
  2761. if (rw == WRITE) {
  2762. if (test_clear_buffer_dirty(bh)) {
  2763. bh->b_end_io = end_buffer_write_sync;
  2764. get_bh(bh);
  2765. submit_bh(WRITE, bh);
  2766. continue;
  2767. }
  2768. } else {
  2769. if (!buffer_uptodate(bh)) {
  2770. bh->b_end_io = end_buffer_read_sync;
  2771. get_bh(bh);
  2772. submit_bh(rw, bh);
  2773. continue;
  2774. }
  2775. }
  2776. unlock_buffer(bh);
  2777. }
  2778. }
  2779. EXPORT_SYMBOL(ll_rw_block);
  2780. void write_dirty_buffer(struct buffer_head *bh, int rw)
  2781. {
  2782. lock_buffer(bh);
  2783. if (!test_clear_buffer_dirty(bh)) {
  2784. unlock_buffer(bh);
  2785. return;
  2786. }
  2787. bh->b_end_io = end_buffer_write_sync;
  2788. get_bh(bh);
  2789. submit_bh(rw, bh);
  2790. }
  2791. EXPORT_SYMBOL(write_dirty_buffer);
  2792. /*
  2793. * For a data-integrity writeout, we need to wait upon any in-progress I/O
  2794. * and then start new I/O and then wait upon it. The caller must have a ref on
  2795. * the buffer_head.
  2796. */
  2797. int __sync_dirty_buffer(struct buffer_head *bh, int rw)
  2798. {
  2799. int ret = 0;
  2800. WARN_ON(atomic_read(&bh->b_count) < 1);
  2801. lock_buffer(bh);
  2802. if (test_clear_buffer_dirty(bh)) {
  2803. get_bh(bh);
  2804. bh->b_end_io = end_buffer_write_sync;
  2805. ret = submit_bh(rw, bh);
  2806. wait_on_buffer(bh);
  2807. if (!ret && !buffer_uptodate(bh))
  2808. ret = -EIO;
  2809. } else {
  2810. unlock_buffer(bh);
  2811. }
  2812. return ret;
  2813. }
  2814. EXPORT_SYMBOL(__sync_dirty_buffer);
  2815. int sync_dirty_buffer(struct buffer_head *bh)
  2816. {
  2817. return __sync_dirty_buffer(bh, WRITE_SYNC);
  2818. }
  2819. EXPORT_SYMBOL(sync_dirty_buffer);
  2820. /*
  2821. * try_to_free_buffers() checks if all the buffers on this particular page
  2822. * are unused, and releases them if so.
  2823. *
  2824. * Exclusion against try_to_free_buffers may be obtained by either
  2825. * locking the page or by holding its mapping's private_lock.
  2826. *
  2827. * If the page is dirty but all the buffers are clean then we need to
  2828. * be sure to mark the page clean as well. This is because the page
  2829. * may be against a block device, and a later reattachment of buffers
  2830. * to a dirty page will set *all* buffers dirty. Which would corrupt
  2831. * filesystem data on the same device.
  2832. *
  2833. * The same applies to regular filesystem pages: if all the buffers are
  2834. * clean then we set the page clean and proceed. To do that, we require
  2835. * total exclusion from __set_page_dirty_buffers(). That is obtained with
  2836. * private_lock.
  2837. *
  2838. * try_to_free_buffers() is non-blocking.
  2839. */
  2840. static inline int buffer_busy(struct buffer_head *bh)
  2841. {
  2842. return atomic_read(&bh->b_count) |
  2843. (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
  2844. }
  2845. static int
  2846. drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
  2847. {
  2848. struct buffer_head *head = page_buffers(page);
  2849. struct buffer_head *bh;
  2850. bh = head;
  2851. do {
  2852. if (buffer_write_io_error(bh) && page->mapping)
  2853. set_bit(AS_EIO, &page->mapping->flags);
  2854. if (buffer_busy(bh))
  2855. goto failed;
  2856. bh = bh->b_this_page;
  2857. } while (bh != head);
  2858. do {
  2859. struct buffer_head *next = bh->b_this_page;
  2860. if (bh->b_assoc_map)
  2861. __remove_assoc_queue(bh);
  2862. bh = next;
  2863. } while (bh != head);
  2864. *buffers_to_free = head;
  2865. __clear_page_buffers(page);
  2866. return 1;
  2867. failed:
  2868. return 0;
  2869. }
  2870. int try_to_free_buffers(struct page *page)
  2871. {
  2872. struct address_space * const mapping = page->mapping;
  2873. struct buffer_head *buffers_to_free = NULL;
  2874. int ret = 0;
  2875. BUG_ON(!PageLocked(page));
  2876. if (PageWriteback(page))
  2877. return 0;
  2878. if (mapping == NULL) { /* can this still happen? */
  2879. ret = drop_buffers(page, &buffers_to_free);
  2880. goto out;
  2881. }
  2882. spin_lock(&mapping->private_lock);
  2883. ret = drop_buffers(page, &buffers_to_free);
  2884. /*
  2885. * If the filesystem writes its buffers by hand (eg ext3)
  2886. * then we can have clean buffers against a dirty page. We
  2887. * clean the page here; otherwise the VM will never notice
  2888. * that the filesystem did any IO at all.
  2889. *
  2890. * Also, during truncate, discard_buffer will have marked all
  2891. * the page's buffers clean. We discover that here and clean
  2892. * the page also.
  2893. *
  2894. * private_lock must be held over this entire operation in order
  2895. * to synchronise against __set_page_dirty_buffers and prevent the
  2896. * dirty bit from being lost.
  2897. */
  2898. if (ret)
  2899. cancel_dirty_page(page, PAGE_CACHE_SIZE);
  2900. spin_unlock(&mapping->private_lock);
  2901. out:
  2902. if (buffers_to_free) {
  2903. struct buffer_head *bh = buffers_to_free;
  2904. do {
  2905. struct buffer_head *next = bh->b_this_page;
  2906. free_buffer_head(bh);
  2907. bh = next;
  2908. } while (bh != buffers_to_free);
  2909. }
  2910. return ret;
  2911. }
  2912. EXPORT_SYMBOL(try_to_free_buffers);
  2913. /*
  2914. * There are no bdflush tunables left. But distributions are
  2915. * still running obsolete flush daemons, so we terminate them here.
  2916. *
  2917. * Use of bdflush() is deprecated and will be removed in a future kernel.
  2918. * The `flush-X' kernel threads fully replace bdflush daemons and this call.
  2919. */
  2920. SYSCALL_DEFINE2(bdflush, int, func, long, data)
  2921. {
  2922. static int msg_count;
  2923. if (!capable(CAP_SYS_ADMIN))
  2924. return -EPERM;
  2925. if (msg_count < 5) {
  2926. msg_count++;
  2927. printk(KERN_INFO
  2928. "warning: process `%s' used the obsolete bdflush"
  2929. " system call\n", current->comm);
  2930. printk(KERN_INFO "Fix your initscripts?\n");
  2931. }
  2932. if (func == 1)
  2933. do_exit(0);
  2934. return 0;
  2935. }
  2936. /*
  2937. * Buffer-head allocation
  2938. */
  2939. static struct kmem_cache *bh_cachep __read_mostly;
  2940. /*
  2941. * Once the number of bh's in the machine exceeds this level, we start
  2942. * stripping them in writeback.
  2943. */
  2944. static unsigned long max_buffer_heads;
  2945. int buffer_heads_over_limit;
  2946. struct bh_accounting {
  2947. int nr; /* Number of live bh's */
  2948. int ratelimit; /* Limit cacheline bouncing */
  2949. };
  2950. static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
  2951. static void recalc_bh_state(void)
  2952. {
  2953. int i;
  2954. int tot = 0;
  2955. if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096)
  2956. return;
  2957. __this_cpu_write(bh_accounting.ratelimit, 0);
  2958. for_each_online_cpu(i)
  2959. tot += per_cpu(bh_accounting, i).nr;
  2960. buffer_heads_over_limit = (tot > max_buffer_heads);
  2961. }
  2962. struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
  2963. {
  2964. struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
  2965. if (ret) {
  2966. INIT_LIST_HEAD(&ret->b_assoc_buffers);
  2967. preempt_disable();
  2968. __this_cpu_inc(bh_accounting.nr);
  2969. recalc_bh_state();
  2970. preempt_enable();
  2971. }
  2972. return ret;
  2973. }
  2974. EXPORT_SYMBOL(alloc_buffer_head);
  2975. void free_buffer_head(struct buffer_head *bh)
  2976. {
  2977. BUG_ON(!list_empty(&bh->b_assoc_buffers));
  2978. kmem_cache_free(bh_cachep, bh);
  2979. preempt_disable();
  2980. __this_cpu_dec(bh_accounting.nr);
  2981. recalc_bh_state();
  2982. preempt_enable();
  2983. }
  2984. EXPORT_SYMBOL(free_buffer_head);
  2985. static void buffer_exit_cpu(int cpu)
  2986. {
  2987. int i;
  2988. struct bh_lru *b = &per_cpu(bh_lrus, cpu);
  2989. for (i = 0; i < BH_LRU_SIZE; i++) {
  2990. brelse(b->bhs[i]);
  2991. b->bhs[i] = NULL;
  2992. }
  2993. this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr);
  2994. per_cpu(bh_accounting, cpu).nr = 0;
  2995. }
  2996. static int buffer_cpu_notify(struct notifier_block *self,
  2997. unsigned long action, void *hcpu)
  2998. {
  2999. if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
  3000. buffer_exit_cpu((unsigned long)hcpu);
  3001. return NOTIFY_OK;
  3002. }
  3003. /**
  3004. * bh_uptodate_or_lock - Test whether the buffer is uptodate
  3005. * @bh: struct buffer_head
  3006. *
  3007. * Return true if the buffer is up-to-date and false,
  3008. * with the buffer locked, if not.
  3009. */
  3010. int bh_uptodate_or_lock(struct buffer_head *bh)
  3011. {
  3012. if (!buffer_uptodate(bh)) {
  3013. lock_buffer(bh);
  3014. if (!buffer_uptodate(bh))
  3015. return 0;
  3016. unlock_buffer(bh);
  3017. }
  3018. return 1;
  3019. }
  3020. EXPORT_SYMBOL(bh_uptodate_or_lock);
  3021. /**
  3022. * bh_submit_read - Submit a locked buffer for reading
  3023. * @bh: struct buffer_head
  3024. *
  3025. * Returns zero on success and -EIO on error.
  3026. */
  3027. int bh_submit_read(struct buffer_head *bh)
  3028. {
  3029. BUG_ON(!buffer_locked(bh));
  3030. if (buffer_uptodate(bh)) {
  3031. unlock_buffer(bh);
  3032. return 0;
  3033. }
  3034. get_bh(bh);
  3035. bh->b_end_io = end_buffer_read_sync;
  3036. submit_bh(READ, bh);
  3037. wait_on_buffer(bh);
  3038. if (buffer_uptodate(bh))
  3039. return 0;
  3040. return -EIO;
  3041. }
  3042. EXPORT_SYMBOL(bh_submit_read);
  3043. void __init buffer_init(void)
  3044. {
  3045. unsigned long nrpages;
  3046. bh_cachep = kmem_cache_create("buffer_head",
  3047. sizeof(struct buffer_head), 0,
  3048. (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
  3049. SLAB_MEM_SPREAD),
  3050. NULL);
  3051. /*
  3052. * Limit the bh occupancy to 10% of ZONE_NORMAL
  3053. */
  3054. nrpages = (nr_free_buffer_pages() * 10) / 100;
  3055. max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
  3056. hotcpu_notifier(buffer_cpu_notify, 0);
  3057. }