extent_io.c 130 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247
  1. #include <linux/bitops.h>
  2. #include <linux/slab.h>
  3. #include <linux/bio.h>
  4. #include <linux/mm.h>
  5. #include <linux/pagemap.h>
  6. #include <linux/page-flags.h>
  7. #include <linux/spinlock.h>
  8. #include <linux/blkdev.h>
  9. #include <linux/swap.h>
  10. #include <linux/writeback.h>
  11. #include <linux/pagevec.h>
  12. #include <linux/prefetch.h>
  13. #include <linux/cleancache.h>
  14. #include "extent_io.h"
  15. #include "extent_map.h"
  16. #include "ctree.h"
  17. #include "btrfs_inode.h"
  18. #include "volumes.h"
  19. #include "check-integrity.h"
  20. #include "locking.h"
  21. #include "rcu-string.h"
  22. #include "backref.h"
  23. static struct kmem_cache *extent_state_cache;
  24. static struct kmem_cache *extent_buffer_cache;
  25. static struct bio_set *btrfs_bioset;
  26. #ifdef CONFIG_BTRFS_DEBUG
  27. static LIST_HEAD(buffers);
  28. static LIST_HEAD(states);
  29. static DEFINE_SPINLOCK(leak_lock);
  30. static inline
  31. void btrfs_leak_debug_add(struct list_head *new, struct list_head *head)
  32. {
  33. unsigned long flags;
  34. spin_lock_irqsave(&leak_lock, flags);
  35. list_add(new, head);
  36. spin_unlock_irqrestore(&leak_lock, flags);
  37. }
  38. static inline
  39. void btrfs_leak_debug_del(struct list_head *entry)
  40. {
  41. unsigned long flags;
  42. spin_lock_irqsave(&leak_lock, flags);
  43. list_del(entry);
  44. spin_unlock_irqrestore(&leak_lock, flags);
  45. }
  46. static inline
  47. void btrfs_leak_debug_check(void)
  48. {
  49. struct extent_state *state;
  50. struct extent_buffer *eb;
  51. while (!list_empty(&states)) {
  52. state = list_entry(states.next, struct extent_state, leak_list);
  53. printk(KERN_ERR "btrfs state leak: start %llu end %llu "
  54. "state %lu in tree %p refs %d\n",
  55. state->start, state->end, state->state, state->tree,
  56. atomic_read(&state->refs));
  57. list_del(&state->leak_list);
  58. kmem_cache_free(extent_state_cache, state);
  59. }
  60. while (!list_empty(&buffers)) {
  61. eb = list_entry(buffers.next, struct extent_buffer, leak_list);
  62. printk(KERN_ERR "btrfs buffer leak start %llu len %lu "
  63. "refs %d\n",
  64. eb->start, eb->len, atomic_read(&eb->refs));
  65. list_del(&eb->leak_list);
  66. kmem_cache_free(extent_buffer_cache, eb);
  67. }
  68. }
  69. #define btrfs_debug_check_extent_io_range(inode, start, end) \
  70. __btrfs_debug_check_extent_io_range(__func__, (inode), (start), (end))
  71. static inline void __btrfs_debug_check_extent_io_range(const char *caller,
  72. struct inode *inode, u64 start, u64 end)
  73. {
  74. u64 isize = i_size_read(inode);
  75. if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) {
  76. printk_ratelimited(KERN_DEBUG
  77. "btrfs: %s: ino %llu isize %llu odd range [%llu,%llu]\n",
  78. caller, btrfs_ino(inode), isize, start, end);
  79. }
  80. }
  81. #else
  82. #define btrfs_leak_debug_add(new, head) do {} while (0)
  83. #define btrfs_leak_debug_del(entry) do {} while (0)
  84. #define btrfs_leak_debug_check() do {} while (0)
  85. #define btrfs_debug_check_extent_io_range(c, s, e) do {} while (0)
  86. #endif
  87. #define BUFFER_LRU_MAX 64
  88. struct tree_entry {
  89. u64 start;
  90. u64 end;
  91. struct rb_node rb_node;
  92. };
  93. struct extent_page_data {
  94. struct bio *bio;
  95. struct extent_io_tree *tree;
  96. get_extent_t *get_extent;
  97. unsigned long bio_flags;
  98. /* tells writepage not to lock the state bits for this range
  99. * it still does the unlocking
  100. */
  101. unsigned int extent_locked:1;
  102. /* tells the submit_bio code to use a WRITE_SYNC */
  103. unsigned int sync_io:1;
  104. };
  105. static noinline void flush_write_bio(void *data);
  106. static inline struct btrfs_fs_info *
  107. tree_fs_info(struct extent_io_tree *tree)
  108. {
  109. return btrfs_sb(tree->mapping->host->i_sb);
  110. }
  111. int __init extent_io_init(void)
  112. {
  113. extent_state_cache = kmem_cache_create("btrfs_extent_state",
  114. sizeof(struct extent_state), 0,
  115. SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
  116. if (!extent_state_cache)
  117. return -ENOMEM;
  118. extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
  119. sizeof(struct extent_buffer), 0,
  120. SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
  121. if (!extent_buffer_cache)
  122. goto free_state_cache;
  123. btrfs_bioset = bioset_create(BIO_POOL_SIZE,
  124. offsetof(struct btrfs_io_bio, bio));
  125. if (!btrfs_bioset)
  126. goto free_buffer_cache;
  127. if (bioset_integrity_create(btrfs_bioset, BIO_POOL_SIZE))
  128. goto free_bioset;
  129. return 0;
  130. free_bioset:
  131. bioset_free(btrfs_bioset);
  132. btrfs_bioset = NULL;
  133. free_buffer_cache:
  134. kmem_cache_destroy(extent_buffer_cache);
  135. extent_buffer_cache = NULL;
  136. free_state_cache:
  137. kmem_cache_destroy(extent_state_cache);
  138. extent_state_cache = NULL;
  139. return -ENOMEM;
  140. }
  141. void extent_io_exit(void)
  142. {
  143. btrfs_leak_debug_check();
  144. /*
  145. * Make sure all delayed rcu free are flushed before we
  146. * destroy caches.
  147. */
  148. rcu_barrier();
  149. if (extent_state_cache)
  150. kmem_cache_destroy(extent_state_cache);
  151. if (extent_buffer_cache)
  152. kmem_cache_destroy(extent_buffer_cache);
  153. if (btrfs_bioset)
  154. bioset_free(btrfs_bioset);
  155. }
  156. void extent_io_tree_init(struct extent_io_tree *tree,
  157. struct address_space *mapping)
  158. {
  159. tree->state = RB_ROOT;
  160. INIT_RADIX_TREE(&tree->buffer, GFP_ATOMIC);
  161. tree->ops = NULL;
  162. tree->dirty_bytes = 0;
  163. spin_lock_init(&tree->lock);
  164. spin_lock_init(&tree->buffer_lock);
  165. tree->mapping = mapping;
  166. }
  167. static struct extent_state *alloc_extent_state(gfp_t mask)
  168. {
  169. struct extent_state *state;
  170. state = kmem_cache_alloc(extent_state_cache, mask);
  171. if (!state)
  172. return state;
  173. state->state = 0;
  174. state->private = 0;
  175. state->tree = NULL;
  176. btrfs_leak_debug_add(&state->leak_list, &states);
  177. atomic_set(&state->refs, 1);
  178. init_waitqueue_head(&state->wq);
  179. trace_alloc_extent_state(state, mask, _RET_IP_);
  180. return state;
  181. }
  182. void free_extent_state(struct extent_state *state)
  183. {
  184. if (!state)
  185. return;
  186. if (atomic_dec_and_test(&state->refs)) {
  187. WARN_ON(state->tree);
  188. btrfs_leak_debug_del(&state->leak_list);
  189. trace_free_extent_state(state, _RET_IP_);
  190. kmem_cache_free(extent_state_cache, state);
  191. }
  192. }
  193. static struct rb_node *tree_insert(struct rb_root *root, u64 offset,
  194. struct rb_node *node)
  195. {
  196. struct rb_node **p = &root->rb_node;
  197. struct rb_node *parent = NULL;
  198. struct tree_entry *entry;
  199. while (*p) {
  200. parent = *p;
  201. entry = rb_entry(parent, struct tree_entry, rb_node);
  202. if (offset < entry->start)
  203. p = &(*p)->rb_left;
  204. else if (offset > entry->end)
  205. p = &(*p)->rb_right;
  206. else
  207. return parent;
  208. }
  209. rb_link_node(node, parent, p);
  210. rb_insert_color(node, root);
  211. return NULL;
  212. }
  213. static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
  214. struct rb_node **prev_ret,
  215. struct rb_node **next_ret)
  216. {
  217. struct rb_root *root = &tree->state;
  218. struct rb_node *n = root->rb_node;
  219. struct rb_node *prev = NULL;
  220. struct rb_node *orig_prev = NULL;
  221. struct tree_entry *entry;
  222. struct tree_entry *prev_entry = NULL;
  223. while (n) {
  224. entry = rb_entry(n, struct tree_entry, rb_node);
  225. prev = n;
  226. prev_entry = entry;
  227. if (offset < entry->start)
  228. n = n->rb_left;
  229. else if (offset > entry->end)
  230. n = n->rb_right;
  231. else
  232. return n;
  233. }
  234. if (prev_ret) {
  235. orig_prev = prev;
  236. while (prev && offset > prev_entry->end) {
  237. prev = rb_next(prev);
  238. prev_entry = rb_entry(prev, struct tree_entry, rb_node);
  239. }
  240. *prev_ret = prev;
  241. prev = orig_prev;
  242. }
  243. if (next_ret) {
  244. prev_entry = rb_entry(prev, struct tree_entry, rb_node);
  245. while (prev && offset < prev_entry->start) {
  246. prev = rb_prev(prev);
  247. prev_entry = rb_entry(prev, struct tree_entry, rb_node);
  248. }
  249. *next_ret = prev;
  250. }
  251. return NULL;
  252. }
  253. static inline struct rb_node *tree_search(struct extent_io_tree *tree,
  254. u64 offset)
  255. {
  256. struct rb_node *prev = NULL;
  257. struct rb_node *ret;
  258. ret = __etree_search(tree, offset, &prev, NULL);
  259. if (!ret)
  260. return prev;
  261. return ret;
  262. }
  263. static void merge_cb(struct extent_io_tree *tree, struct extent_state *new,
  264. struct extent_state *other)
  265. {
  266. if (tree->ops && tree->ops->merge_extent_hook)
  267. tree->ops->merge_extent_hook(tree->mapping->host, new,
  268. other);
  269. }
  270. /*
  271. * utility function to look for merge candidates inside a given range.
  272. * Any extents with matching state are merged together into a single
  273. * extent in the tree. Extents with EXTENT_IO in their state field
  274. * are not merged because the end_io handlers need to be able to do
  275. * operations on them without sleeping (or doing allocations/splits).
  276. *
  277. * This should be called with the tree lock held.
  278. */
  279. static void merge_state(struct extent_io_tree *tree,
  280. struct extent_state *state)
  281. {
  282. struct extent_state *other;
  283. struct rb_node *other_node;
  284. if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY))
  285. return;
  286. other_node = rb_prev(&state->rb_node);
  287. if (other_node) {
  288. other = rb_entry(other_node, struct extent_state, rb_node);
  289. if (other->end == state->start - 1 &&
  290. other->state == state->state) {
  291. merge_cb(tree, state, other);
  292. state->start = other->start;
  293. other->tree = NULL;
  294. rb_erase(&other->rb_node, &tree->state);
  295. free_extent_state(other);
  296. }
  297. }
  298. other_node = rb_next(&state->rb_node);
  299. if (other_node) {
  300. other = rb_entry(other_node, struct extent_state, rb_node);
  301. if (other->start == state->end + 1 &&
  302. other->state == state->state) {
  303. merge_cb(tree, state, other);
  304. state->end = other->end;
  305. other->tree = NULL;
  306. rb_erase(&other->rb_node, &tree->state);
  307. free_extent_state(other);
  308. }
  309. }
  310. }
  311. static void set_state_cb(struct extent_io_tree *tree,
  312. struct extent_state *state, unsigned long *bits)
  313. {
  314. if (tree->ops && tree->ops->set_bit_hook)
  315. tree->ops->set_bit_hook(tree->mapping->host, state, bits);
  316. }
  317. static void clear_state_cb(struct extent_io_tree *tree,
  318. struct extent_state *state, unsigned long *bits)
  319. {
  320. if (tree->ops && tree->ops->clear_bit_hook)
  321. tree->ops->clear_bit_hook(tree->mapping->host, state, bits);
  322. }
  323. static void set_state_bits(struct extent_io_tree *tree,
  324. struct extent_state *state, unsigned long *bits);
  325. /*
  326. * insert an extent_state struct into the tree. 'bits' are set on the
  327. * struct before it is inserted.
  328. *
  329. * This may return -EEXIST if the extent is already there, in which case the
  330. * state struct is freed.
  331. *
  332. * The tree lock is not taken internally. This is a utility function and
  333. * probably isn't what you want to call (see set/clear_extent_bit).
  334. */
  335. static int insert_state(struct extent_io_tree *tree,
  336. struct extent_state *state, u64 start, u64 end,
  337. unsigned long *bits)
  338. {
  339. struct rb_node *node;
  340. if (end < start)
  341. WARN(1, KERN_ERR "btrfs end < start %llu %llu\n",
  342. end, start);
  343. state->start = start;
  344. state->end = end;
  345. set_state_bits(tree, state, bits);
  346. node = tree_insert(&tree->state, end, &state->rb_node);
  347. if (node) {
  348. struct extent_state *found;
  349. found = rb_entry(node, struct extent_state, rb_node);
  350. printk(KERN_ERR "btrfs found node %llu %llu on insert of "
  351. "%llu %llu\n",
  352. found->start, found->end, start, end);
  353. return -EEXIST;
  354. }
  355. state->tree = tree;
  356. merge_state(tree, state);
  357. return 0;
  358. }
  359. static void split_cb(struct extent_io_tree *tree, struct extent_state *orig,
  360. u64 split)
  361. {
  362. if (tree->ops && tree->ops->split_extent_hook)
  363. tree->ops->split_extent_hook(tree->mapping->host, orig, split);
  364. }
  365. /*
  366. * split a given extent state struct in two, inserting the preallocated
  367. * struct 'prealloc' as the newly created second half. 'split' indicates an
  368. * offset inside 'orig' where it should be split.
  369. *
  370. * Before calling,
  371. * the tree has 'orig' at [orig->start, orig->end]. After calling, there
  372. * are two extent state structs in the tree:
  373. * prealloc: [orig->start, split - 1]
  374. * orig: [ split, orig->end ]
  375. *
  376. * The tree locks are not taken by this function. They need to be held
  377. * by the caller.
  378. */
  379. static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
  380. struct extent_state *prealloc, u64 split)
  381. {
  382. struct rb_node *node;
  383. split_cb(tree, orig, split);
  384. prealloc->start = orig->start;
  385. prealloc->end = split - 1;
  386. prealloc->state = orig->state;
  387. orig->start = split;
  388. node = tree_insert(&tree->state, prealloc->end, &prealloc->rb_node);
  389. if (node) {
  390. free_extent_state(prealloc);
  391. return -EEXIST;
  392. }
  393. prealloc->tree = tree;
  394. return 0;
  395. }
  396. static struct extent_state *next_state(struct extent_state *state)
  397. {
  398. struct rb_node *next = rb_next(&state->rb_node);
  399. if (next)
  400. return rb_entry(next, struct extent_state, rb_node);
  401. else
  402. return NULL;
  403. }
  404. /*
  405. * utility function to clear some bits in an extent state struct.
  406. * it will optionally wake up any one waiting on this state (wake == 1).
  407. *
  408. * If no bits are set on the state struct after clearing things, the
  409. * struct is freed and removed from the tree
  410. */
  411. static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
  412. struct extent_state *state,
  413. unsigned long *bits, int wake)
  414. {
  415. struct extent_state *next;
  416. unsigned long bits_to_clear = *bits & ~EXTENT_CTLBITS;
  417. if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
  418. u64 range = state->end - state->start + 1;
  419. WARN_ON(range > tree->dirty_bytes);
  420. tree->dirty_bytes -= range;
  421. }
  422. clear_state_cb(tree, state, bits);
  423. state->state &= ~bits_to_clear;
  424. if (wake)
  425. wake_up(&state->wq);
  426. if (state->state == 0) {
  427. next = next_state(state);
  428. if (state->tree) {
  429. rb_erase(&state->rb_node, &tree->state);
  430. state->tree = NULL;
  431. free_extent_state(state);
  432. } else {
  433. WARN_ON(1);
  434. }
  435. } else {
  436. merge_state(tree, state);
  437. next = next_state(state);
  438. }
  439. return next;
  440. }
  441. static struct extent_state *
  442. alloc_extent_state_atomic(struct extent_state *prealloc)
  443. {
  444. if (!prealloc)
  445. prealloc = alloc_extent_state(GFP_ATOMIC);
  446. return prealloc;
  447. }
  448. static void extent_io_tree_panic(struct extent_io_tree *tree, int err)
  449. {
  450. btrfs_panic(tree_fs_info(tree), err, "Locking error: "
  451. "Extent tree was modified by another "
  452. "thread while locked.");
  453. }
  454. /*
  455. * clear some bits on a range in the tree. This may require splitting
  456. * or inserting elements in the tree, so the gfp mask is used to
  457. * indicate which allocations or sleeping are allowed.
  458. *
  459. * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
  460. * the given range from the tree regardless of state (ie for truncate).
  461. *
  462. * the range [start, end] is inclusive.
  463. *
  464. * This takes the tree lock, and returns 0 on success and < 0 on error.
  465. */
  466. int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
  467. unsigned long bits, int wake, int delete,
  468. struct extent_state **cached_state,
  469. gfp_t mask)
  470. {
  471. struct extent_state *state;
  472. struct extent_state *cached;
  473. struct extent_state *prealloc = NULL;
  474. struct rb_node *node;
  475. u64 last_end;
  476. int err;
  477. int clear = 0;
  478. btrfs_debug_check_extent_io_range(tree->mapping->host, start, end);
  479. if (bits & EXTENT_DELALLOC)
  480. bits |= EXTENT_NORESERVE;
  481. if (delete)
  482. bits |= ~EXTENT_CTLBITS;
  483. bits |= EXTENT_FIRST_DELALLOC;
  484. if (bits & (EXTENT_IOBITS | EXTENT_BOUNDARY))
  485. clear = 1;
  486. again:
  487. if (!prealloc && (mask & __GFP_WAIT)) {
  488. prealloc = alloc_extent_state(mask);
  489. if (!prealloc)
  490. return -ENOMEM;
  491. }
  492. spin_lock(&tree->lock);
  493. if (cached_state) {
  494. cached = *cached_state;
  495. if (clear) {
  496. *cached_state = NULL;
  497. cached_state = NULL;
  498. }
  499. if (cached && cached->tree && cached->start <= start &&
  500. cached->end > start) {
  501. if (clear)
  502. atomic_dec(&cached->refs);
  503. state = cached;
  504. goto hit_next;
  505. }
  506. if (clear)
  507. free_extent_state(cached);
  508. }
  509. /*
  510. * this search will find the extents that end after
  511. * our range starts
  512. */
  513. node = tree_search(tree, start);
  514. if (!node)
  515. goto out;
  516. state = rb_entry(node, struct extent_state, rb_node);
  517. hit_next:
  518. if (state->start > end)
  519. goto out;
  520. WARN_ON(state->end < start);
  521. last_end = state->end;
  522. /* the state doesn't have the wanted bits, go ahead */
  523. if (!(state->state & bits)) {
  524. state = next_state(state);
  525. goto next;
  526. }
  527. /*
  528. * | ---- desired range ---- |
  529. * | state | or
  530. * | ------------- state -------------- |
  531. *
  532. * We need to split the extent we found, and may flip
  533. * bits on second half.
  534. *
  535. * If the extent we found extends past our range, we
  536. * just split and search again. It'll get split again
  537. * the next time though.
  538. *
  539. * If the extent we found is inside our range, we clear
  540. * the desired bit on it.
  541. */
  542. if (state->start < start) {
  543. prealloc = alloc_extent_state_atomic(prealloc);
  544. BUG_ON(!prealloc);
  545. err = split_state(tree, state, prealloc, start);
  546. if (err)
  547. extent_io_tree_panic(tree, err);
  548. prealloc = NULL;
  549. if (err)
  550. goto out;
  551. if (state->end <= end) {
  552. state = clear_state_bit(tree, state, &bits, wake);
  553. goto next;
  554. }
  555. goto search_again;
  556. }
  557. /*
  558. * | ---- desired range ---- |
  559. * | state |
  560. * We need to split the extent, and clear the bit
  561. * on the first half
  562. */
  563. if (state->start <= end && state->end > end) {
  564. prealloc = alloc_extent_state_atomic(prealloc);
  565. BUG_ON(!prealloc);
  566. err = split_state(tree, state, prealloc, end + 1);
  567. if (err)
  568. extent_io_tree_panic(tree, err);
  569. if (wake)
  570. wake_up(&state->wq);
  571. clear_state_bit(tree, prealloc, &bits, wake);
  572. prealloc = NULL;
  573. goto out;
  574. }
  575. state = clear_state_bit(tree, state, &bits, wake);
  576. next:
  577. if (last_end == (u64)-1)
  578. goto out;
  579. start = last_end + 1;
  580. if (start <= end && state && !need_resched())
  581. goto hit_next;
  582. goto search_again;
  583. out:
  584. spin_unlock(&tree->lock);
  585. if (prealloc)
  586. free_extent_state(prealloc);
  587. return 0;
  588. search_again:
  589. if (start > end)
  590. goto out;
  591. spin_unlock(&tree->lock);
  592. if (mask & __GFP_WAIT)
  593. cond_resched();
  594. goto again;
  595. }
  596. static void wait_on_state(struct extent_io_tree *tree,
  597. struct extent_state *state)
  598. __releases(tree->lock)
  599. __acquires(tree->lock)
  600. {
  601. DEFINE_WAIT(wait);
  602. prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
  603. spin_unlock(&tree->lock);
  604. schedule();
  605. spin_lock(&tree->lock);
  606. finish_wait(&state->wq, &wait);
  607. }
  608. /*
  609. * waits for one or more bits to clear on a range in the state tree.
  610. * The range [start, end] is inclusive.
  611. * The tree lock is taken by this function
  612. */
  613. static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
  614. unsigned long bits)
  615. {
  616. struct extent_state *state;
  617. struct rb_node *node;
  618. btrfs_debug_check_extent_io_range(tree->mapping->host, start, end);
  619. spin_lock(&tree->lock);
  620. again:
  621. while (1) {
  622. /*
  623. * this search will find all the extents that end after
  624. * our range starts
  625. */
  626. node = tree_search(tree, start);
  627. if (!node)
  628. break;
  629. state = rb_entry(node, struct extent_state, rb_node);
  630. if (state->start > end)
  631. goto out;
  632. if (state->state & bits) {
  633. start = state->start;
  634. atomic_inc(&state->refs);
  635. wait_on_state(tree, state);
  636. free_extent_state(state);
  637. goto again;
  638. }
  639. start = state->end + 1;
  640. if (start > end)
  641. break;
  642. cond_resched_lock(&tree->lock);
  643. }
  644. out:
  645. spin_unlock(&tree->lock);
  646. }
  647. static void set_state_bits(struct extent_io_tree *tree,
  648. struct extent_state *state,
  649. unsigned long *bits)
  650. {
  651. unsigned long bits_to_set = *bits & ~EXTENT_CTLBITS;
  652. set_state_cb(tree, state, bits);
  653. if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
  654. u64 range = state->end - state->start + 1;
  655. tree->dirty_bytes += range;
  656. }
  657. state->state |= bits_to_set;
  658. }
  659. static void cache_state(struct extent_state *state,
  660. struct extent_state **cached_ptr)
  661. {
  662. if (cached_ptr && !(*cached_ptr)) {
  663. if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY)) {
  664. *cached_ptr = state;
  665. atomic_inc(&state->refs);
  666. }
  667. }
  668. }
  669. /*
  670. * set some bits on a range in the tree. This may require allocations or
  671. * sleeping, so the gfp mask is used to indicate what is allowed.
  672. *
  673. * If any of the exclusive bits are set, this will fail with -EEXIST if some
  674. * part of the range already has the desired bits set. The start of the
  675. * existing range is returned in failed_start in this case.
  676. *
  677. * [start, end] is inclusive This takes the tree lock.
  678. */
  679. static int __must_check
  680. __set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
  681. unsigned long bits, unsigned long exclusive_bits,
  682. u64 *failed_start, struct extent_state **cached_state,
  683. gfp_t mask)
  684. {
  685. struct extent_state *state;
  686. struct extent_state *prealloc = NULL;
  687. struct rb_node *node;
  688. int err = 0;
  689. u64 last_start;
  690. u64 last_end;
  691. btrfs_debug_check_extent_io_range(tree->mapping->host, start, end);
  692. bits |= EXTENT_FIRST_DELALLOC;
  693. again:
  694. if (!prealloc && (mask & __GFP_WAIT)) {
  695. prealloc = alloc_extent_state(mask);
  696. BUG_ON(!prealloc);
  697. }
  698. spin_lock(&tree->lock);
  699. if (cached_state && *cached_state) {
  700. state = *cached_state;
  701. if (state->start <= start && state->end > start &&
  702. state->tree) {
  703. node = &state->rb_node;
  704. goto hit_next;
  705. }
  706. }
  707. /*
  708. * this search will find all the extents that end after
  709. * our range starts.
  710. */
  711. node = tree_search(tree, start);
  712. if (!node) {
  713. prealloc = alloc_extent_state_atomic(prealloc);
  714. BUG_ON(!prealloc);
  715. err = insert_state(tree, prealloc, start, end, &bits);
  716. if (err)
  717. extent_io_tree_panic(tree, err);
  718. prealloc = NULL;
  719. goto out;
  720. }
  721. state = rb_entry(node, struct extent_state, rb_node);
  722. hit_next:
  723. last_start = state->start;
  724. last_end = state->end;
  725. /*
  726. * | ---- desired range ---- |
  727. * | state |
  728. *
  729. * Just lock what we found and keep going
  730. */
  731. if (state->start == start && state->end <= end) {
  732. if (state->state & exclusive_bits) {
  733. *failed_start = state->start;
  734. err = -EEXIST;
  735. goto out;
  736. }
  737. set_state_bits(tree, state, &bits);
  738. cache_state(state, cached_state);
  739. merge_state(tree, state);
  740. if (last_end == (u64)-1)
  741. goto out;
  742. start = last_end + 1;
  743. state = next_state(state);
  744. if (start < end && state && state->start == start &&
  745. !need_resched())
  746. goto hit_next;
  747. goto search_again;
  748. }
  749. /*
  750. * | ---- desired range ---- |
  751. * | state |
  752. * or
  753. * | ------------- state -------------- |
  754. *
  755. * We need to split the extent we found, and may flip bits on
  756. * second half.
  757. *
  758. * If the extent we found extends past our
  759. * range, we just split and search again. It'll get split
  760. * again the next time though.
  761. *
  762. * If the extent we found is inside our range, we set the
  763. * desired bit on it.
  764. */
  765. if (state->start < start) {
  766. if (state->state & exclusive_bits) {
  767. *failed_start = start;
  768. err = -EEXIST;
  769. goto out;
  770. }
  771. prealloc = alloc_extent_state_atomic(prealloc);
  772. BUG_ON(!prealloc);
  773. err = split_state(tree, state, prealloc, start);
  774. if (err)
  775. extent_io_tree_panic(tree, err);
  776. prealloc = NULL;
  777. if (err)
  778. goto out;
  779. if (state->end <= end) {
  780. set_state_bits(tree, state, &bits);
  781. cache_state(state, cached_state);
  782. merge_state(tree, state);
  783. if (last_end == (u64)-1)
  784. goto out;
  785. start = last_end + 1;
  786. state = next_state(state);
  787. if (start < end && state && state->start == start &&
  788. !need_resched())
  789. goto hit_next;
  790. }
  791. goto search_again;
  792. }
  793. /*
  794. * | ---- desired range ---- |
  795. * | state | or | state |
  796. *
  797. * There's a hole, we need to insert something in it and
  798. * ignore the extent we found.
  799. */
  800. if (state->start > start) {
  801. u64 this_end;
  802. if (end < last_start)
  803. this_end = end;
  804. else
  805. this_end = last_start - 1;
  806. prealloc = alloc_extent_state_atomic(prealloc);
  807. BUG_ON(!prealloc);
  808. /*
  809. * Avoid to free 'prealloc' if it can be merged with
  810. * the later extent.
  811. */
  812. err = insert_state(tree, prealloc, start, this_end,
  813. &bits);
  814. if (err)
  815. extent_io_tree_panic(tree, err);
  816. cache_state(prealloc, cached_state);
  817. prealloc = NULL;
  818. start = this_end + 1;
  819. goto search_again;
  820. }
  821. /*
  822. * | ---- desired range ---- |
  823. * | state |
  824. * We need to split the extent, and set the bit
  825. * on the first half
  826. */
  827. if (state->start <= end && state->end > end) {
  828. if (state->state & exclusive_bits) {
  829. *failed_start = start;
  830. err = -EEXIST;
  831. goto out;
  832. }
  833. prealloc = alloc_extent_state_atomic(prealloc);
  834. BUG_ON(!prealloc);
  835. err = split_state(tree, state, prealloc, end + 1);
  836. if (err)
  837. extent_io_tree_panic(tree, err);
  838. set_state_bits(tree, prealloc, &bits);
  839. cache_state(prealloc, cached_state);
  840. merge_state(tree, prealloc);
  841. prealloc = NULL;
  842. goto out;
  843. }
  844. goto search_again;
  845. out:
  846. spin_unlock(&tree->lock);
  847. if (prealloc)
  848. free_extent_state(prealloc);
  849. return err;
  850. search_again:
  851. if (start > end)
  852. goto out;
  853. spin_unlock(&tree->lock);
  854. if (mask & __GFP_WAIT)
  855. cond_resched();
  856. goto again;
  857. }
  858. int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
  859. unsigned long bits, u64 * failed_start,
  860. struct extent_state **cached_state, gfp_t mask)
  861. {
  862. return __set_extent_bit(tree, start, end, bits, 0, failed_start,
  863. cached_state, mask);
  864. }
  865. /**
  866. * convert_extent_bit - convert all bits in a given range from one bit to
  867. * another
  868. * @tree: the io tree to search
  869. * @start: the start offset in bytes
  870. * @end: the end offset in bytes (inclusive)
  871. * @bits: the bits to set in this range
  872. * @clear_bits: the bits to clear in this range
  873. * @cached_state: state that we're going to cache
  874. * @mask: the allocation mask
  875. *
  876. * This will go through and set bits for the given range. If any states exist
  877. * already in this range they are set with the given bit and cleared of the
  878. * clear_bits. This is only meant to be used by things that are mergeable, ie
  879. * converting from say DELALLOC to DIRTY. This is not meant to be used with
  880. * boundary bits like LOCK.
  881. */
  882. int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
  883. unsigned long bits, unsigned long clear_bits,
  884. struct extent_state **cached_state, gfp_t mask)
  885. {
  886. struct extent_state *state;
  887. struct extent_state *prealloc = NULL;
  888. struct rb_node *node;
  889. int err = 0;
  890. u64 last_start;
  891. u64 last_end;
  892. btrfs_debug_check_extent_io_range(tree->mapping->host, start, end);
  893. again:
  894. if (!prealloc && (mask & __GFP_WAIT)) {
  895. prealloc = alloc_extent_state(mask);
  896. if (!prealloc)
  897. return -ENOMEM;
  898. }
  899. spin_lock(&tree->lock);
  900. if (cached_state && *cached_state) {
  901. state = *cached_state;
  902. if (state->start <= start && state->end > start &&
  903. state->tree) {
  904. node = &state->rb_node;
  905. goto hit_next;
  906. }
  907. }
  908. /*
  909. * this search will find all the extents that end after
  910. * our range starts.
  911. */
  912. node = tree_search(tree, start);
  913. if (!node) {
  914. prealloc = alloc_extent_state_atomic(prealloc);
  915. if (!prealloc) {
  916. err = -ENOMEM;
  917. goto out;
  918. }
  919. err = insert_state(tree, prealloc, start, end, &bits);
  920. prealloc = NULL;
  921. if (err)
  922. extent_io_tree_panic(tree, err);
  923. goto out;
  924. }
  925. state = rb_entry(node, struct extent_state, rb_node);
  926. hit_next:
  927. last_start = state->start;
  928. last_end = state->end;
  929. /*
  930. * | ---- desired range ---- |
  931. * | state |
  932. *
  933. * Just lock what we found and keep going
  934. */
  935. if (state->start == start && state->end <= end) {
  936. set_state_bits(tree, state, &bits);
  937. cache_state(state, cached_state);
  938. state = clear_state_bit(tree, state, &clear_bits, 0);
  939. if (last_end == (u64)-1)
  940. goto out;
  941. start = last_end + 1;
  942. if (start < end && state && state->start == start &&
  943. !need_resched())
  944. goto hit_next;
  945. goto search_again;
  946. }
  947. /*
  948. * | ---- desired range ---- |
  949. * | state |
  950. * or
  951. * | ------------- state -------------- |
  952. *
  953. * We need to split the extent we found, and may flip bits on
  954. * second half.
  955. *
  956. * If the extent we found extends past our
  957. * range, we just split and search again. It'll get split
  958. * again the next time though.
  959. *
  960. * If the extent we found is inside our range, we set the
  961. * desired bit on it.
  962. */
  963. if (state->start < start) {
  964. prealloc = alloc_extent_state_atomic(prealloc);
  965. if (!prealloc) {
  966. err = -ENOMEM;
  967. goto out;
  968. }
  969. err = split_state(tree, state, prealloc, start);
  970. if (err)
  971. extent_io_tree_panic(tree, err);
  972. prealloc = NULL;
  973. if (err)
  974. goto out;
  975. if (state->end <= end) {
  976. set_state_bits(tree, state, &bits);
  977. cache_state(state, cached_state);
  978. state = clear_state_bit(tree, state, &clear_bits, 0);
  979. if (last_end == (u64)-1)
  980. goto out;
  981. start = last_end + 1;
  982. if (start < end && state && state->start == start &&
  983. !need_resched())
  984. goto hit_next;
  985. }
  986. goto search_again;
  987. }
  988. /*
  989. * | ---- desired range ---- |
  990. * | state | or | state |
  991. *
  992. * There's a hole, we need to insert something in it and
  993. * ignore the extent we found.
  994. */
  995. if (state->start > start) {
  996. u64 this_end;
  997. if (end < last_start)
  998. this_end = end;
  999. else
  1000. this_end = last_start - 1;
  1001. prealloc = alloc_extent_state_atomic(prealloc);
  1002. if (!prealloc) {
  1003. err = -ENOMEM;
  1004. goto out;
  1005. }
  1006. /*
  1007. * Avoid to free 'prealloc' if it can be merged with
  1008. * the later extent.
  1009. */
  1010. err = insert_state(tree, prealloc, start, this_end,
  1011. &bits);
  1012. if (err)
  1013. extent_io_tree_panic(tree, err);
  1014. cache_state(prealloc, cached_state);
  1015. prealloc = NULL;
  1016. start = this_end + 1;
  1017. goto search_again;
  1018. }
  1019. /*
  1020. * | ---- desired range ---- |
  1021. * | state |
  1022. * We need to split the extent, and set the bit
  1023. * on the first half
  1024. */
  1025. if (state->start <= end && state->end > end) {
  1026. prealloc = alloc_extent_state_atomic(prealloc);
  1027. if (!prealloc) {
  1028. err = -ENOMEM;
  1029. goto out;
  1030. }
  1031. err = split_state(tree, state, prealloc, end + 1);
  1032. if (err)
  1033. extent_io_tree_panic(tree, err);
  1034. set_state_bits(tree, prealloc, &bits);
  1035. cache_state(prealloc, cached_state);
  1036. clear_state_bit(tree, prealloc, &clear_bits, 0);
  1037. prealloc = NULL;
  1038. goto out;
  1039. }
  1040. goto search_again;
  1041. out:
  1042. spin_unlock(&tree->lock);
  1043. if (prealloc)
  1044. free_extent_state(prealloc);
  1045. return err;
  1046. search_again:
  1047. if (start > end)
  1048. goto out;
  1049. spin_unlock(&tree->lock);
  1050. if (mask & __GFP_WAIT)
  1051. cond_resched();
  1052. goto again;
  1053. }
  1054. /* wrappers around set/clear extent bit */
  1055. int set_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
  1056. gfp_t mask)
  1057. {
  1058. return set_extent_bit(tree, start, end, EXTENT_DIRTY, NULL,
  1059. NULL, mask);
  1060. }
  1061. int set_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
  1062. unsigned long bits, gfp_t mask)
  1063. {
  1064. return set_extent_bit(tree, start, end, bits, NULL,
  1065. NULL, mask);
  1066. }
  1067. int clear_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
  1068. unsigned long bits, gfp_t mask)
  1069. {
  1070. return clear_extent_bit(tree, start, end, bits, 0, 0, NULL, mask);
  1071. }
  1072. int set_extent_delalloc(struct extent_io_tree *tree, u64 start, u64 end,
  1073. struct extent_state **cached_state, gfp_t mask)
  1074. {
  1075. return set_extent_bit(tree, start, end,
  1076. EXTENT_DELALLOC | EXTENT_UPTODATE,
  1077. NULL, cached_state, mask);
  1078. }
  1079. int set_extent_defrag(struct extent_io_tree *tree, u64 start, u64 end,
  1080. struct extent_state **cached_state, gfp_t mask)
  1081. {
  1082. return set_extent_bit(tree, start, end,
  1083. EXTENT_DELALLOC | EXTENT_UPTODATE | EXTENT_DEFRAG,
  1084. NULL, cached_state, mask);
  1085. }
  1086. int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
  1087. gfp_t mask)
  1088. {
  1089. return clear_extent_bit(tree, start, end,
  1090. EXTENT_DIRTY | EXTENT_DELALLOC |
  1091. EXTENT_DO_ACCOUNTING, 0, 0, NULL, mask);
  1092. }
  1093. int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
  1094. gfp_t mask)
  1095. {
  1096. return set_extent_bit(tree, start, end, EXTENT_NEW, NULL,
  1097. NULL, mask);
  1098. }
  1099. int set_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
  1100. struct extent_state **cached_state, gfp_t mask)
  1101. {
  1102. return set_extent_bit(tree, start, end, EXTENT_UPTODATE, NULL,
  1103. cached_state, mask);
  1104. }
  1105. int clear_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
  1106. struct extent_state **cached_state, gfp_t mask)
  1107. {
  1108. return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0,
  1109. cached_state, mask);
  1110. }
  1111. /*
  1112. * either insert or lock state struct between start and end use mask to tell
  1113. * us if waiting is desired.
  1114. */
  1115. int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
  1116. unsigned long bits, struct extent_state **cached_state)
  1117. {
  1118. int err;
  1119. u64 failed_start;
  1120. while (1) {
  1121. err = __set_extent_bit(tree, start, end, EXTENT_LOCKED | bits,
  1122. EXTENT_LOCKED, &failed_start,
  1123. cached_state, GFP_NOFS);
  1124. if (err == -EEXIST) {
  1125. wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
  1126. start = failed_start;
  1127. } else
  1128. break;
  1129. WARN_ON(start > end);
  1130. }
  1131. return err;
  1132. }
  1133. int lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
  1134. {
  1135. return lock_extent_bits(tree, start, end, 0, NULL);
  1136. }
  1137. int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
  1138. {
  1139. int err;
  1140. u64 failed_start;
  1141. err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
  1142. &failed_start, NULL, GFP_NOFS);
  1143. if (err == -EEXIST) {
  1144. if (failed_start > start)
  1145. clear_extent_bit(tree, start, failed_start - 1,
  1146. EXTENT_LOCKED, 1, 0, NULL, GFP_NOFS);
  1147. return 0;
  1148. }
  1149. return 1;
  1150. }
  1151. int unlock_extent_cached(struct extent_io_tree *tree, u64 start, u64 end,
  1152. struct extent_state **cached, gfp_t mask)
  1153. {
  1154. return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, cached,
  1155. mask);
  1156. }
  1157. int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end)
  1158. {
  1159. return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, NULL,
  1160. GFP_NOFS);
  1161. }
  1162. int extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
  1163. {
  1164. unsigned long index = start >> PAGE_CACHE_SHIFT;
  1165. unsigned long end_index = end >> PAGE_CACHE_SHIFT;
  1166. struct page *page;
  1167. while (index <= end_index) {
  1168. page = find_get_page(inode->i_mapping, index);
  1169. BUG_ON(!page); /* Pages should be in the extent_io_tree */
  1170. clear_page_dirty_for_io(page);
  1171. page_cache_release(page);
  1172. index++;
  1173. }
  1174. return 0;
  1175. }
  1176. int extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end)
  1177. {
  1178. unsigned long index = start >> PAGE_CACHE_SHIFT;
  1179. unsigned long end_index = end >> PAGE_CACHE_SHIFT;
  1180. struct page *page;
  1181. while (index <= end_index) {
  1182. page = find_get_page(inode->i_mapping, index);
  1183. BUG_ON(!page); /* Pages should be in the extent_io_tree */
  1184. account_page_redirty(page);
  1185. __set_page_dirty_nobuffers(page);
  1186. page_cache_release(page);
  1187. index++;
  1188. }
  1189. return 0;
  1190. }
  1191. /*
  1192. * helper function to set both pages and extents in the tree writeback
  1193. */
  1194. static int set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
  1195. {
  1196. unsigned long index = start >> PAGE_CACHE_SHIFT;
  1197. unsigned long end_index = end >> PAGE_CACHE_SHIFT;
  1198. struct page *page;
  1199. while (index <= end_index) {
  1200. page = find_get_page(tree->mapping, index);
  1201. BUG_ON(!page); /* Pages should be in the extent_io_tree */
  1202. set_page_writeback(page);
  1203. page_cache_release(page);
  1204. index++;
  1205. }
  1206. return 0;
  1207. }
  1208. /* find the first state struct with 'bits' set after 'start', and
  1209. * return it. tree->lock must be held. NULL will returned if
  1210. * nothing was found after 'start'
  1211. */
  1212. static struct extent_state *
  1213. find_first_extent_bit_state(struct extent_io_tree *tree,
  1214. u64 start, unsigned long bits)
  1215. {
  1216. struct rb_node *node;
  1217. struct extent_state *state;
  1218. /*
  1219. * this search will find all the extents that end after
  1220. * our range starts.
  1221. */
  1222. node = tree_search(tree, start);
  1223. if (!node)
  1224. goto out;
  1225. while (1) {
  1226. state = rb_entry(node, struct extent_state, rb_node);
  1227. if (state->end >= start && (state->state & bits))
  1228. return state;
  1229. node = rb_next(node);
  1230. if (!node)
  1231. break;
  1232. }
  1233. out:
  1234. return NULL;
  1235. }
  1236. /*
  1237. * find the first offset in the io tree with 'bits' set. zero is
  1238. * returned if we find something, and *start_ret and *end_ret are
  1239. * set to reflect the state struct that was found.
  1240. *
  1241. * If nothing was found, 1 is returned. If found something, return 0.
  1242. */
  1243. int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
  1244. u64 *start_ret, u64 *end_ret, unsigned long bits,
  1245. struct extent_state **cached_state)
  1246. {
  1247. struct extent_state *state;
  1248. struct rb_node *n;
  1249. int ret = 1;
  1250. spin_lock(&tree->lock);
  1251. if (cached_state && *cached_state) {
  1252. state = *cached_state;
  1253. if (state->end == start - 1 && state->tree) {
  1254. n = rb_next(&state->rb_node);
  1255. while (n) {
  1256. state = rb_entry(n, struct extent_state,
  1257. rb_node);
  1258. if (state->state & bits)
  1259. goto got_it;
  1260. n = rb_next(n);
  1261. }
  1262. free_extent_state(*cached_state);
  1263. *cached_state = NULL;
  1264. goto out;
  1265. }
  1266. free_extent_state(*cached_state);
  1267. *cached_state = NULL;
  1268. }
  1269. state = find_first_extent_bit_state(tree, start, bits);
  1270. got_it:
  1271. if (state) {
  1272. cache_state(state, cached_state);
  1273. *start_ret = state->start;
  1274. *end_ret = state->end;
  1275. ret = 0;
  1276. }
  1277. out:
  1278. spin_unlock(&tree->lock);
  1279. return ret;
  1280. }
  1281. /*
  1282. * find a contiguous range of bytes in the file marked as delalloc, not
  1283. * more than 'max_bytes'. start and end are used to return the range,
  1284. *
  1285. * 1 is returned if we find something, 0 if nothing was in the tree
  1286. */
  1287. static noinline u64 find_delalloc_range(struct extent_io_tree *tree,
  1288. u64 *start, u64 *end, u64 max_bytes,
  1289. struct extent_state **cached_state)
  1290. {
  1291. struct rb_node *node;
  1292. struct extent_state *state;
  1293. u64 cur_start = *start;
  1294. u64 found = 0;
  1295. u64 total_bytes = 0;
  1296. spin_lock(&tree->lock);
  1297. /*
  1298. * this search will find all the extents that end after
  1299. * our range starts.
  1300. */
  1301. node = tree_search(tree, cur_start);
  1302. if (!node) {
  1303. if (!found)
  1304. *end = (u64)-1;
  1305. goto out;
  1306. }
  1307. while (1) {
  1308. state = rb_entry(node, struct extent_state, rb_node);
  1309. if (found && (state->start != cur_start ||
  1310. (state->state & EXTENT_BOUNDARY))) {
  1311. goto out;
  1312. }
  1313. if (!(state->state & EXTENT_DELALLOC)) {
  1314. if (!found)
  1315. *end = state->end;
  1316. goto out;
  1317. }
  1318. if (!found) {
  1319. *start = state->start;
  1320. *cached_state = state;
  1321. atomic_inc(&state->refs);
  1322. }
  1323. found++;
  1324. *end = state->end;
  1325. cur_start = state->end + 1;
  1326. node = rb_next(node);
  1327. total_bytes += state->end - state->start + 1;
  1328. if (total_bytes >= max_bytes)
  1329. break;
  1330. if (!node)
  1331. break;
  1332. }
  1333. out:
  1334. spin_unlock(&tree->lock);
  1335. return found;
  1336. }
  1337. static noinline void __unlock_for_delalloc(struct inode *inode,
  1338. struct page *locked_page,
  1339. u64 start, u64 end)
  1340. {
  1341. int ret;
  1342. struct page *pages[16];
  1343. unsigned long index = start >> PAGE_CACHE_SHIFT;
  1344. unsigned long end_index = end >> PAGE_CACHE_SHIFT;
  1345. unsigned long nr_pages = end_index - index + 1;
  1346. int i;
  1347. if (index == locked_page->index && end_index == index)
  1348. return;
  1349. while (nr_pages > 0) {
  1350. ret = find_get_pages_contig(inode->i_mapping, index,
  1351. min_t(unsigned long, nr_pages,
  1352. ARRAY_SIZE(pages)), pages);
  1353. for (i = 0; i < ret; i++) {
  1354. if (pages[i] != locked_page)
  1355. unlock_page(pages[i]);
  1356. page_cache_release(pages[i]);
  1357. }
  1358. nr_pages -= ret;
  1359. index += ret;
  1360. cond_resched();
  1361. }
  1362. }
  1363. static noinline int lock_delalloc_pages(struct inode *inode,
  1364. struct page *locked_page,
  1365. u64 delalloc_start,
  1366. u64 delalloc_end)
  1367. {
  1368. unsigned long index = delalloc_start >> PAGE_CACHE_SHIFT;
  1369. unsigned long start_index = index;
  1370. unsigned long end_index = delalloc_end >> PAGE_CACHE_SHIFT;
  1371. unsigned long pages_locked = 0;
  1372. struct page *pages[16];
  1373. unsigned long nrpages;
  1374. int ret;
  1375. int i;
  1376. /* the caller is responsible for locking the start index */
  1377. if (index == locked_page->index && index == end_index)
  1378. return 0;
  1379. /* skip the page at the start index */
  1380. nrpages = end_index - index + 1;
  1381. while (nrpages > 0) {
  1382. ret = find_get_pages_contig(inode->i_mapping, index,
  1383. min_t(unsigned long,
  1384. nrpages, ARRAY_SIZE(pages)), pages);
  1385. if (ret == 0) {
  1386. ret = -EAGAIN;
  1387. goto done;
  1388. }
  1389. /* now we have an array of pages, lock them all */
  1390. for (i = 0; i < ret; i++) {
  1391. /*
  1392. * the caller is taking responsibility for
  1393. * locked_page
  1394. */
  1395. if (pages[i] != locked_page) {
  1396. lock_page(pages[i]);
  1397. if (!PageDirty(pages[i]) ||
  1398. pages[i]->mapping != inode->i_mapping) {
  1399. ret = -EAGAIN;
  1400. unlock_page(pages[i]);
  1401. page_cache_release(pages[i]);
  1402. goto done;
  1403. }
  1404. }
  1405. page_cache_release(pages[i]);
  1406. pages_locked++;
  1407. }
  1408. nrpages -= ret;
  1409. index += ret;
  1410. cond_resched();
  1411. }
  1412. ret = 0;
  1413. done:
  1414. if (ret && pages_locked) {
  1415. __unlock_for_delalloc(inode, locked_page,
  1416. delalloc_start,
  1417. ((u64)(start_index + pages_locked - 1)) <<
  1418. PAGE_CACHE_SHIFT);
  1419. }
  1420. return ret;
  1421. }
  1422. /*
  1423. * find a contiguous range of bytes in the file marked as delalloc, not
  1424. * more than 'max_bytes'. start and end are used to return the range,
  1425. *
  1426. * 1 is returned if we find something, 0 if nothing was in the tree
  1427. */
  1428. STATIC u64 find_lock_delalloc_range(struct inode *inode,
  1429. struct extent_io_tree *tree,
  1430. struct page *locked_page, u64 *start,
  1431. u64 *end, u64 max_bytes)
  1432. {
  1433. u64 delalloc_start;
  1434. u64 delalloc_end;
  1435. u64 found;
  1436. struct extent_state *cached_state = NULL;
  1437. int ret;
  1438. int loops = 0;
  1439. again:
  1440. /* step one, find a bunch of delalloc bytes starting at start */
  1441. delalloc_start = *start;
  1442. delalloc_end = 0;
  1443. found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
  1444. max_bytes, &cached_state);
  1445. if (!found || delalloc_end <= *start) {
  1446. *start = delalloc_start;
  1447. *end = delalloc_end;
  1448. free_extent_state(cached_state);
  1449. return 0;
  1450. }
  1451. /*
  1452. * start comes from the offset of locked_page. We have to lock
  1453. * pages in order, so we can't process delalloc bytes before
  1454. * locked_page
  1455. */
  1456. if (delalloc_start < *start)
  1457. delalloc_start = *start;
  1458. /*
  1459. * make sure to limit the number of pages we try to lock down
  1460. */
  1461. if (delalloc_end + 1 - delalloc_start > max_bytes)
  1462. delalloc_end = delalloc_start + max_bytes - 1;
  1463. /* step two, lock all the pages after the page that has start */
  1464. ret = lock_delalloc_pages(inode, locked_page,
  1465. delalloc_start, delalloc_end);
  1466. if (ret == -EAGAIN) {
  1467. /* some of the pages are gone, lets avoid looping by
  1468. * shortening the size of the delalloc range we're searching
  1469. */
  1470. free_extent_state(cached_state);
  1471. if (!loops) {
  1472. max_bytes = PAGE_CACHE_SIZE;
  1473. loops = 1;
  1474. goto again;
  1475. } else {
  1476. found = 0;
  1477. goto out_failed;
  1478. }
  1479. }
  1480. BUG_ON(ret); /* Only valid values are 0 and -EAGAIN */
  1481. /* step three, lock the state bits for the whole range */
  1482. lock_extent_bits(tree, delalloc_start, delalloc_end, 0, &cached_state);
  1483. /* then test to make sure it is all still delalloc */
  1484. ret = test_range_bit(tree, delalloc_start, delalloc_end,
  1485. EXTENT_DELALLOC, 1, cached_state);
  1486. if (!ret) {
  1487. unlock_extent_cached(tree, delalloc_start, delalloc_end,
  1488. &cached_state, GFP_NOFS);
  1489. __unlock_for_delalloc(inode, locked_page,
  1490. delalloc_start, delalloc_end);
  1491. cond_resched();
  1492. goto again;
  1493. }
  1494. free_extent_state(cached_state);
  1495. *start = delalloc_start;
  1496. *end = delalloc_end;
  1497. out_failed:
  1498. return found;
  1499. }
  1500. int extent_clear_unlock_delalloc(struct inode *inode, u64 start, u64 end,
  1501. struct page *locked_page,
  1502. unsigned long clear_bits,
  1503. unsigned long page_ops)
  1504. {
  1505. struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
  1506. int ret;
  1507. struct page *pages[16];
  1508. unsigned long index = start >> PAGE_CACHE_SHIFT;
  1509. unsigned long end_index = end >> PAGE_CACHE_SHIFT;
  1510. unsigned long nr_pages = end_index - index + 1;
  1511. int i;
  1512. clear_extent_bit(tree, start, end, clear_bits, 1, 0, NULL, GFP_NOFS);
  1513. if (page_ops == 0)
  1514. return 0;
  1515. while (nr_pages > 0) {
  1516. ret = find_get_pages_contig(inode->i_mapping, index,
  1517. min_t(unsigned long,
  1518. nr_pages, ARRAY_SIZE(pages)), pages);
  1519. for (i = 0; i < ret; i++) {
  1520. if (page_ops & PAGE_SET_PRIVATE2)
  1521. SetPagePrivate2(pages[i]);
  1522. if (pages[i] == locked_page) {
  1523. page_cache_release(pages[i]);
  1524. continue;
  1525. }
  1526. if (page_ops & PAGE_CLEAR_DIRTY)
  1527. clear_page_dirty_for_io(pages[i]);
  1528. if (page_ops & PAGE_SET_WRITEBACK)
  1529. set_page_writeback(pages[i]);
  1530. if (page_ops & PAGE_END_WRITEBACK)
  1531. end_page_writeback(pages[i]);
  1532. if (page_ops & PAGE_UNLOCK)
  1533. unlock_page(pages[i]);
  1534. page_cache_release(pages[i]);
  1535. }
  1536. nr_pages -= ret;
  1537. index += ret;
  1538. cond_resched();
  1539. }
  1540. return 0;
  1541. }
  1542. /*
  1543. * count the number of bytes in the tree that have a given bit(s)
  1544. * set. This can be fairly slow, except for EXTENT_DIRTY which is
  1545. * cached. The total number found is returned.
  1546. */
  1547. u64 count_range_bits(struct extent_io_tree *tree,
  1548. u64 *start, u64 search_end, u64 max_bytes,
  1549. unsigned long bits, int contig)
  1550. {
  1551. struct rb_node *node;
  1552. struct extent_state *state;
  1553. u64 cur_start = *start;
  1554. u64 total_bytes = 0;
  1555. u64 last = 0;
  1556. int found = 0;
  1557. if (WARN_ON(search_end <= cur_start))
  1558. return 0;
  1559. spin_lock(&tree->lock);
  1560. if (cur_start == 0 && bits == EXTENT_DIRTY) {
  1561. total_bytes = tree->dirty_bytes;
  1562. goto out;
  1563. }
  1564. /*
  1565. * this search will find all the extents that end after
  1566. * our range starts.
  1567. */
  1568. node = tree_search(tree, cur_start);
  1569. if (!node)
  1570. goto out;
  1571. while (1) {
  1572. state = rb_entry(node, struct extent_state, rb_node);
  1573. if (state->start > search_end)
  1574. break;
  1575. if (contig && found && state->start > last + 1)
  1576. break;
  1577. if (state->end >= cur_start && (state->state & bits) == bits) {
  1578. total_bytes += min(search_end, state->end) + 1 -
  1579. max(cur_start, state->start);
  1580. if (total_bytes >= max_bytes)
  1581. break;
  1582. if (!found) {
  1583. *start = max(cur_start, state->start);
  1584. found = 1;
  1585. }
  1586. last = state->end;
  1587. } else if (contig && found) {
  1588. break;
  1589. }
  1590. node = rb_next(node);
  1591. if (!node)
  1592. break;
  1593. }
  1594. out:
  1595. spin_unlock(&tree->lock);
  1596. return total_bytes;
  1597. }
  1598. /*
  1599. * set the private field for a given byte offset in the tree. If there isn't
  1600. * an extent_state there already, this does nothing.
  1601. */
  1602. static int set_state_private(struct extent_io_tree *tree, u64 start, u64 private)
  1603. {
  1604. struct rb_node *node;
  1605. struct extent_state *state;
  1606. int ret = 0;
  1607. spin_lock(&tree->lock);
  1608. /*
  1609. * this search will find all the extents that end after
  1610. * our range starts.
  1611. */
  1612. node = tree_search(tree, start);
  1613. if (!node) {
  1614. ret = -ENOENT;
  1615. goto out;
  1616. }
  1617. state = rb_entry(node, struct extent_state, rb_node);
  1618. if (state->start != start) {
  1619. ret = -ENOENT;
  1620. goto out;
  1621. }
  1622. state->private = private;
  1623. out:
  1624. spin_unlock(&tree->lock);
  1625. return ret;
  1626. }
  1627. int get_state_private(struct extent_io_tree *tree, u64 start, u64 *private)
  1628. {
  1629. struct rb_node *node;
  1630. struct extent_state *state;
  1631. int ret = 0;
  1632. spin_lock(&tree->lock);
  1633. /*
  1634. * this search will find all the extents that end after
  1635. * our range starts.
  1636. */
  1637. node = tree_search(tree, start);
  1638. if (!node) {
  1639. ret = -ENOENT;
  1640. goto out;
  1641. }
  1642. state = rb_entry(node, struct extent_state, rb_node);
  1643. if (state->start != start) {
  1644. ret = -ENOENT;
  1645. goto out;
  1646. }
  1647. *private = state->private;
  1648. out:
  1649. spin_unlock(&tree->lock);
  1650. return ret;
  1651. }
  1652. /*
  1653. * searches a range in the state tree for a given mask.
  1654. * If 'filled' == 1, this returns 1 only if every extent in the tree
  1655. * has the bits set. Otherwise, 1 is returned if any bit in the
  1656. * range is found set.
  1657. */
  1658. int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
  1659. unsigned long bits, int filled, struct extent_state *cached)
  1660. {
  1661. struct extent_state *state = NULL;
  1662. struct rb_node *node;
  1663. int bitset = 0;
  1664. spin_lock(&tree->lock);
  1665. if (cached && cached->tree && cached->start <= start &&
  1666. cached->end > start)
  1667. node = &cached->rb_node;
  1668. else
  1669. node = tree_search(tree, start);
  1670. while (node && start <= end) {
  1671. state = rb_entry(node, struct extent_state, rb_node);
  1672. if (filled && state->start > start) {
  1673. bitset = 0;
  1674. break;
  1675. }
  1676. if (state->start > end)
  1677. break;
  1678. if (state->state & bits) {
  1679. bitset = 1;
  1680. if (!filled)
  1681. break;
  1682. } else if (filled) {
  1683. bitset = 0;
  1684. break;
  1685. }
  1686. if (state->end == (u64)-1)
  1687. break;
  1688. start = state->end + 1;
  1689. if (start > end)
  1690. break;
  1691. node = rb_next(node);
  1692. if (!node) {
  1693. if (filled)
  1694. bitset = 0;
  1695. break;
  1696. }
  1697. }
  1698. spin_unlock(&tree->lock);
  1699. return bitset;
  1700. }
  1701. /*
  1702. * helper function to set a given page up to date if all the
  1703. * extents in the tree for that page are up to date
  1704. */
  1705. static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)
  1706. {
  1707. u64 start = page_offset(page);
  1708. u64 end = start + PAGE_CACHE_SIZE - 1;
  1709. if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
  1710. SetPageUptodate(page);
  1711. }
  1712. /*
  1713. * When IO fails, either with EIO or csum verification fails, we
  1714. * try other mirrors that might have a good copy of the data. This
  1715. * io_failure_record is used to record state as we go through all the
  1716. * mirrors. If another mirror has good data, the page is set up to date
  1717. * and things continue. If a good mirror can't be found, the original
  1718. * bio end_io callback is called to indicate things have failed.
  1719. */
  1720. struct io_failure_record {
  1721. struct page *page;
  1722. u64 start;
  1723. u64 len;
  1724. u64 logical;
  1725. unsigned long bio_flags;
  1726. int this_mirror;
  1727. int failed_mirror;
  1728. int in_validation;
  1729. };
  1730. static int free_io_failure(struct inode *inode, struct io_failure_record *rec,
  1731. int did_repair)
  1732. {
  1733. int ret;
  1734. int err = 0;
  1735. struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
  1736. set_state_private(failure_tree, rec->start, 0);
  1737. ret = clear_extent_bits(failure_tree, rec->start,
  1738. rec->start + rec->len - 1,
  1739. EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
  1740. if (ret)
  1741. err = ret;
  1742. ret = clear_extent_bits(&BTRFS_I(inode)->io_tree, rec->start,
  1743. rec->start + rec->len - 1,
  1744. EXTENT_DAMAGED, GFP_NOFS);
  1745. if (ret && !err)
  1746. err = ret;
  1747. kfree(rec);
  1748. return err;
  1749. }
  1750. static void repair_io_failure_callback(struct bio *bio, int err)
  1751. {
  1752. complete(bio->bi_private);
  1753. }
  1754. /*
  1755. * this bypasses the standard btrfs submit functions deliberately, as
  1756. * the standard behavior is to write all copies in a raid setup. here we only
  1757. * want to write the one bad copy. so we do the mapping for ourselves and issue
  1758. * submit_bio directly.
  1759. * to avoid any synchronization issues, wait for the data after writing, which
  1760. * actually prevents the read that triggered the error from finishing.
  1761. * currently, there can be no more than two copies of every data bit. thus,
  1762. * exactly one rewrite is required.
  1763. */
  1764. int repair_io_failure(struct btrfs_fs_info *fs_info, u64 start,
  1765. u64 length, u64 logical, struct page *page,
  1766. int mirror_num)
  1767. {
  1768. struct bio *bio;
  1769. struct btrfs_device *dev;
  1770. DECLARE_COMPLETION_ONSTACK(compl);
  1771. u64 map_length = 0;
  1772. u64 sector;
  1773. struct btrfs_bio *bbio = NULL;
  1774. struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree;
  1775. int ret;
  1776. ASSERT(!(fs_info->sb->s_flags & MS_RDONLY));
  1777. BUG_ON(!mirror_num);
  1778. /* we can't repair anything in raid56 yet */
  1779. if (btrfs_is_parity_mirror(map_tree, logical, length, mirror_num))
  1780. return 0;
  1781. bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
  1782. if (!bio)
  1783. return -EIO;
  1784. bio->bi_private = &compl;
  1785. bio->bi_end_io = repair_io_failure_callback;
  1786. bio->bi_size = 0;
  1787. map_length = length;
  1788. ret = btrfs_map_block(fs_info, WRITE, logical,
  1789. &map_length, &bbio, mirror_num);
  1790. if (ret) {
  1791. bio_put(bio);
  1792. return -EIO;
  1793. }
  1794. BUG_ON(mirror_num != bbio->mirror_num);
  1795. sector = bbio->stripes[mirror_num-1].physical >> 9;
  1796. bio->bi_sector = sector;
  1797. dev = bbio->stripes[mirror_num-1].dev;
  1798. kfree(bbio);
  1799. if (!dev || !dev->bdev || !dev->writeable) {
  1800. bio_put(bio);
  1801. return -EIO;
  1802. }
  1803. bio->bi_bdev = dev->bdev;
  1804. bio_add_page(bio, page, length, start - page_offset(page));
  1805. btrfsic_submit_bio(WRITE_SYNC, bio);
  1806. wait_for_completion(&compl);
  1807. if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) {
  1808. /* try to remap that extent elsewhere? */
  1809. bio_put(bio);
  1810. btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
  1811. return -EIO;
  1812. }
  1813. printk_ratelimited_in_rcu(KERN_INFO "btrfs read error corrected: ino %lu off %llu "
  1814. "(dev %s sector %llu)\n", page->mapping->host->i_ino,
  1815. start, rcu_str_deref(dev->name), sector);
  1816. bio_put(bio);
  1817. return 0;
  1818. }
  1819. int repair_eb_io_failure(struct btrfs_root *root, struct extent_buffer *eb,
  1820. int mirror_num)
  1821. {
  1822. u64 start = eb->start;
  1823. unsigned long i, num_pages = num_extent_pages(eb->start, eb->len);
  1824. int ret = 0;
  1825. if (root->fs_info->sb->s_flags & MS_RDONLY)
  1826. return -EROFS;
  1827. for (i = 0; i < num_pages; i++) {
  1828. struct page *p = extent_buffer_page(eb, i);
  1829. ret = repair_io_failure(root->fs_info, start, PAGE_CACHE_SIZE,
  1830. start, p, mirror_num);
  1831. if (ret)
  1832. break;
  1833. start += PAGE_CACHE_SIZE;
  1834. }
  1835. return ret;
  1836. }
  1837. /*
  1838. * each time an IO finishes, we do a fast check in the IO failure tree
  1839. * to see if we need to process or clean up an io_failure_record
  1840. */
  1841. static int clean_io_failure(u64 start, struct page *page)
  1842. {
  1843. u64 private;
  1844. u64 private_failure;
  1845. struct io_failure_record *failrec;
  1846. struct inode *inode = page->mapping->host;
  1847. struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
  1848. struct extent_state *state;
  1849. int num_copies;
  1850. int did_repair = 0;
  1851. int ret;
  1852. private = 0;
  1853. ret = count_range_bits(&BTRFS_I(inode)->io_failure_tree, &private,
  1854. (u64)-1, 1, EXTENT_DIRTY, 0);
  1855. if (!ret)
  1856. return 0;
  1857. ret = get_state_private(&BTRFS_I(inode)->io_failure_tree, start,
  1858. &private_failure);
  1859. if (ret)
  1860. return 0;
  1861. failrec = (struct io_failure_record *)(unsigned long) private_failure;
  1862. BUG_ON(!failrec->this_mirror);
  1863. if (failrec->in_validation) {
  1864. /* there was no real error, just free the record */
  1865. pr_debug("clean_io_failure: freeing dummy error at %llu\n",
  1866. failrec->start);
  1867. did_repair = 1;
  1868. goto out;
  1869. }
  1870. if (fs_info->sb->s_flags & MS_RDONLY)
  1871. goto out;
  1872. spin_lock(&BTRFS_I(inode)->io_tree.lock);
  1873. state = find_first_extent_bit_state(&BTRFS_I(inode)->io_tree,
  1874. failrec->start,
  1875. EXTENT_LOCKED);
  1876. spin_unlock(&BTRFS_I(inode)->io_tree.lock);
  1877. if (state && state->start <= failrec->start &&
  1878. state->end >= failrec->start + failrec->len - 1) {
  1879. num_copies = btrfs_num_copies(fs_info, failrec->logical,
  1880. failrec->len);
  1881. if (num_copies > 1) {
  1882. ret = repair_io_failure(fs_info, start, failrec->len,
  1883. failrec->logical, page,
  1884. failrec->failed_mirror);
  1885. did_repair = !ret;
  1886. }
  1887. ret = 0;
  1888. }
  1889. out:
  1890. if (!ret)
  1891. ret = free_io_failure(inode, failrec, did_repair);
  1892. return ret;
  1893. }
  1894. /*
  1895. * this is a generic handler for readpage errors (default
  1896. * readpage_io_failed_hook). if other copies exist, read those and write back
  1897. * good data to the failed position. does not investigate in remapping the
  1898. * failed extent elsewhere, hoping the device will be smart enough to do this as
  1899. * needed
  1900. */
  1901. static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset,
  1902. struct page *page, u64 start, u64 end,
  1903. int failed_mirror)
  1904. {
  1905. struct io_failure_record *failrec = NULL;
  1906. u64 private;
  1907. struct extent_map *em;
  1908. struct inode *inode = page->mapping->host;
  1909. struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
  1910. struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
  1911. struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
  1912. struct bio *bio;
  1913. struct btrfs_io_bio *btrfs_failed_bio;
  1914. struct btrfs_io_bio *btrfs_bio;
  1915. int num_copies;
  1916. int ret;
  1917. int read_mode;
  1918. u64 logical;
  1919. BUG_ON(failed_bio->bi_rw & REQ_WRITE);
  1920. ret = get_state_private(failure_tree, start, &private);
  1921. if (ret) {
  1922. failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
  1923. if (!failrec)
  1924. return -ENOMEM;
  1925. failrec->start = start;
  1926. failrec->len = end - start + 1;
  1927. failrec->this_mirror = 0;
  1928. failrec->bio_flags = 0;
  1929. failrec->in_validation = 0;
  1930. read_lock(&em_tree->lock);
  1931. em = lookup_extent_mapping(em_tree, start, failrec->len);
  1932. if (!em) {
  1933. read_unlock(&em_tree->lock);
  1934. kfree(failrec);
  1935. return -EIO;
  1936. }
  1937. if (em->start > start || em->start + em->len < start) {
  1938. free_extent_map(em);
  1939. em = NULL;
  1940. }
  1941. read_unlock(&em_tree->lock);
  1942. if (!em) {
  1943. kfree(failrec);
  1944. return -EIO;
  1945. }
  1946. logical = start - em->start;
  1947. logical = em->block_start + logical;
  1948. if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
  1949. logical = em->block_start;
  1950. failrec->bio_flags = EXTENT_BIO_COMPRESSED;
  1951. extent_set_compress_type(&failrec->bio_flags,
  1952. em->compress_type);
  1953. }
  1954. pr_debug("bio_readpage_error: (new) logical=%llu, start=%llu, "
  1955. "len=%llu\n", logical, start, failrec->len);
  1956. failrec->logical = logical;
  1957. free_extent_map(em);
  1958. /* set the bits in the private failure tree */
  1959. ret = set_extent_bits(failure_tree, start, end,
  1960. EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
  1961. if (ret >= 0)
  1962. ret = set_state_private(failure_tree, start,
  1963. (u64)(unsigned long)failrec);
  1964. /* set the bits in the inode's tree */
  1965. if (ret >= 0)
  1966. ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED,
  1967. GFP_NOFS);
  1968. if (ret < 0) {
  1969. kfree(failrec);
  1970. return ret;
  1971. }
  1972. } else {
  1973. failrec = (struct io_failure_record *)(unsigned long)private;
  1974. pr_debug("bio_readpage_error: (found) logical=%llu, "
  1975. "start=%llu, len=%llu, validation=%d\n",
  1976. failrec->logical, failrec->start, failrec->len,
  1977. failrec->in_validation);
  1978. /*
  1979. * when data can be on disk more than twice, add to failrec here
  1980. * (e.g. with a list for failed_mirror) to make
  1981. * clean_io_failure() clean all those errors at once.
  1982. */
  1983. }
  1984. num_copies = btrfs_num_copies(BTRFS_I(inode)->root->fs_info,
  1985. failrec->logical, failrec->len);
  1986. if (num_copies == 1) {
  1987. /*
  1988. * we only have a single copy of the data, so don't bother with
  1989. * all the retry and error correction code that follows. no
  1990. * matter what the error is, it is very likely to persist.
  1991. */
  1992. pr_debug("bio_readpage_error: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d\n",
  1993. num_copies, failrec->this_mirror, failed_mirror);
  1994. free_io_failure(inode, failrec, 0);
  1995. return -EIO;
  1996. }
  1997. /*
  1998. * there are two premises:
  1999. * a) deliver good data to the caller
  2000. * b) correct the bad sectors on disk
  2001. */
  2002. if (failed_bio->bi_vcnt > 1) {
  2003. /*
  2004. * to fulfill b), we need to know the exact failing sectors, as
  2005. * we don't want to rewrite any more than the failed ones. thus,
  2006. * we need separate read requests for the failed bio
  2007. *
  2008. * if the following BUG_ON triggers, our validation request got
  2009. * merged. we need separate requests for our algorithm to work.
  2010. */
  2011. BUG_ON(failrec->in_validation);
  2012. failrec->in_validation = 1;
  2013. failrec->this_mirror = failed_mirror;
  2014. read_mode = READ_SYNC | REQ_FAILFAST_DEV;
  2015. } else {
  2016. /*
  2017. * we're ready to fulfill a) and b) alongside. get a good copy
  2018. * of the failed sector and if we succeed, we have setup
  2019. * everything for repair_io_failure to do the rest for us.
  2020. */
  2021. if (failrec->in_validation) {
  2022. BUG_ON(failrec->this_mirror != failed_mirror);
  2023. failrec->in_validation = 0;
  2024. failrec->this_mirror = 0;
  2025. }
  2026. failrec->failed_mirror = failed_mirror;
  2027. failrec->this_mirror++;
  2028. if (failrec->this_mirror == failed_mirror)
  2029. failrec->this_mirror++;
  2030. read_mode = READ_SYNC;
  2031. }
  2032. if (failrec->this_mirror > num_copies) {
  2033. pr_debug("bio_readpage_error: (fail) num_copies=%d, next_mirror %d, failed_mirror %d\n",
  2034. num_copies, failrec->this_mirror, failed_mirror);
  2035. free_io_failure(inode, failrec, 0);
  2036. return -EIO;
  2037. }
  2038. bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
  2039. if (!bio) {
  2040. free_io_failure(inode, failrec, 0);
  2041. return -EIO;
  2042. }
  2043. bio->bi_end_io = failed_bio->bi_end_io;
  2044. bio->bi_sector = failrec->logical >> 9;
  2045. bio->bi_bdev = BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev;
  2046. bio->bi_size = 0;
  2047. btrfs_failed_bio = btrfs_io_bio(failed_bio);
  2048. if (btrfs_failed_bio->csum) {
  2049. struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
  2050. u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
  2051. btrfs_bio = btrfs_io_bio(bio);
  2052. btrfs_bio->csum = btrfs_bio->csum_inline;
  2053. phy_offset >>= inode->i_sb->s_blocksize_bits;
  2054. phy_offset *= csum_size;
  2055. memcpy(btrfs_bio->csum, btrfs_failed_bio->csum + phy_offset,
  2056. csum_size);
  2057. }
  2058. bio_add_page(bio, page, failrec->len, start - page_offset(page));
  2059. pr_debug("bio_readpage_error: submitting new read[%#x] to "
  2060. "this_mirror=%d, num_copies=%d, in_validation=%d\n", read_mode,
  2061. failrec->this_mirror, num_copies, failrec->in_validation);
  2062. ret = tree->ops->submit_bio_hook(inode, read_mode, bio,
  2063. failrec->this_mirror,
  2064. failrec->bio_flags, 0);
  2065. return ret;
  2066. }
  2067. /* lots and lots of room for performance fixes in the end_bio funcs */
  2068. int end_extent_writepage(struct page *page, int err, u64 start, u64 end)
  2069. {
  2070. int uptodate = (err == 0);
  2071. struct extent_io_tree *tree;
  2072. int ret;
  2073. tree = &BTRFS_I(page->mapping->host)->io_tree;
  2074. if (tree->ops && tree->ops->writepage_end_io_hook) {
  2075. ret = tree->ops->writepage_end_io_hook(page, start,
  2076. end, NULL, uptodate);
  2077. if (ret)
  2078. uptodate = 0;
  2079. }
  2080. if (!uptodate) {
  2081. ClearPageUptodate(page);
  2082. SetPageError(page);
  2083. }
  2084. return 0;
  2085. }
  2086. /*
  2087. * after a writepage IO is done, we need to:
  2088. * clear the uptodate bits on error
  2089. * clear the writeback bits in the extent tree for this IO
  2090. * end_page_writeback if the page has no more pending IO
  2091. *
  2092. * Scheduling is not allowed, so the extent state tree is expected
  2093. * to have one and only one object corresponding to this IO.
  2094. */
  2095. static void end_bio_extent_writepage(struct bio *bio, int err)
  2096. {
  2097. struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
  2098. struct extent_io_tree *tree;
  2099. u64 start;
  2100. u64 end;
  2101. do {
  2102. struct page *page = bvec->bv_page;
  2103. tree = &BTRFS_I(page->mapping->host)->io_tree;
  2104. /* We always issue full-page reads, but if some block
  2105. * in a page fails to read, blk_update_request() will
  2106. * advance bv_offset and adjust bv_len to compensate.
  2107. * Print a warning for nonzero offsets, and an error
  2108. * if they don't add up to a full page. */
  2109. if (bvec->bv_offset || bvec->bv_len != PAGE_CACHE_SIZE)
  2110. printk("%s page write in btrfs with offset %u and length %u\n",
  2111. bvec->bv_offset + bvec->bv_len != PAGE_CACHE_SIZE
  2112. ? KERN_ERR "partial" : KERN_INFO "incomplete",
  2113. bvec->bv_offset, bvec->bv_len);
  2114. start = page_offset(page);
  2115. end = start + bvec->bv_offset + bvec->bv_len - 1;
  2116. if (--bvec >= bio->bi_io_vec)
  2117. prefetchw(&bvec->bv_page->flags);
  2118. if (end_extent_writepage(page, err, start, end))
  2119. continue;
  2120. end_page_writeback(page);
  2121. } while (bvec >= bio->bi_io_vec);
  2122. bio_put(bio);
  2123. }
  2124. static void
  2125. endio_readpage_release_extent(struct extent_io_tree *tree, u64 start, u64 len,
  2126. int uptodate)
  2127. {
  2128. struct extent_state *cached = NULL;
  2129. u64 end = start + len - 1;
  2130. if (uptodate && tree->track_uptodate)
  2131. set_extent_uptodate(tree, start, end, &cached, GFP_ATOMIC);
  2132. unlock_extent_cached(tree, start, end, &cached, GFP_ATOMIC);
  2133. }
  2134. /*
  2135. * after a readpage IO is done, we need to:
  2136. * clear the uptodate bits on error
  2137. * set the uptodate bits if things worked
  2138. * set the page up to date if all extents in the tree are uptodate
  2139. * clear the lock bit in the extent tree
  2140. * unlock the page if there are no other extents locked for it
  2141. *
  2142. * Scheduling is not allowed, so the extent state tree is expected
  2143. * to have one and only one object corresponding to this IO.
  2144. */
  2145. static void end_bio_extent_readpage(struct bio *bio, int err)
  2146. {
  2147. int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
  2148. struct bio_vec *bvec_end = bio->bi_io_vec + bio->bi_vcnt - 1;
  2149. struct bio_vec *bvec = bio->bi_io_vec;
  2150. struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
  2151. struct extent_io_tree *tree;
  2152. u64 offset = 0;
  2153. u64 start;
  2154. u64 end;
  2155. u64 len;
  2156. u64 extent_start = 0;
  2157. u64 extent_len = 0;
  2158. int mirror;
  2159. int ret;
  2160. if (err)
  2161. uptodate = 0;
  2162. do {
  2163. struct page *page = bvec->bv_page;
  2164. struct inode *inode = page->mapping->host;
  2165. pr_debug("end_bio_extent_readpage: bi_sector=%llu, err=%d, "
  2166. "mirror=%lu\n", (u64)bio->bi_sector, err,
  2167. io_bio->mirror_num);
  2168. tree = &BTRFS_I(inode)->io_tree;
  2169. /* We always issue full-page reads, but if some block
  2170. * in a page fails to read, blk_update_request() will
  2171. * advance bv_offset and adjust bv_len to compensate.
  2172. * Print a warning for nonzero offsets, and an error
  2173. * if they don't add up to a full page. */
  2174. if (bvec->bv_offset || bvec->bv_len != PAGE_CACHE_SIZE)
  2175. printk("%s page read in btrfs with offset %u and length %u\n",
  2176. bvec->bv_offset + bvec->bv_len != PAGE_CACHE_SIZE
  2177. ? KERN_ERR "partial" : KERN_INFO "incomplete",
  2178. bvec->bv_offset, bvec->bv_len);
  2179. start = page_offset(page);
  2180. end = start + bvec->bv_offset + bvec->bv_len - 1;
  2181. len = bvec->bv_len;
  2182. if (++bvec <= bvec_end)
  2183. prefetchw(&bvec->bv_page->flags);
  2184. mirror = io_bio->mirror_num;
  2185. if (likely(uptodate && tree->ops &&
  2186. tree->ops->readpage_end_io_hook)) {
  2187. ret = tree->ops->readpage_end_io_hook(io_bio, offset,
  2188. page, start, end,
  2189. mirror);
  2190. if (ret)
  2191. uptodate = 0;
  2192. else
  2193. clean_io_failure(start, page);
  2194. }
  2195. if (likely(uptodate))
  2196. goto readpage_ok;
  2197. if (tree->ops && tree->ops->readpage_io_failed_hook) {
  2198. ret = tree->ops->readpage_io_failed_hook(page, mirror);
  2199. if (!ret && !err &&
  2200. test_bit(BIO_UPTODATE, &bio->bi_flags))
  2201. uptodate = 1;
  2202. } else {
  2203. /*
  2204. * The generic bio_readpage_error handles errors the
  2205. * following way: If possible, new read requests are
  2206. * created and submitted and will end up in
  2207. * end_bio_extent_readpage as well (if we're lucky, not
  2208. * in the !uptodate case). In that case it returns 0 and
  2209. * we just go on with the next page in our bio. If it
  2210. * can't handle the error it will return -EIO and we
  2211. * remain responsible for that page.
  2212. */
  2213. ret = bio_readpage_error(bio, offset, page, start, end,
  2214. mirror);
  2215. if (ret == 0) {
  2216. uptodate =
  2217. test_bit(BIO_UPTODATE, &bio->bi_flags);
  2218. if (err)
  2219. uptodate = 0;
  2220. continue;
  2221. }
  2222. }
  2223. readpage_ok:
  2224. if (likely(uptodate)) {
  2225. loff_t i_size = i_size_read(inode);
  2226. pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
  2227. unsigned offset;
  2228. /* Zero out the end if this page straddles i_size */
  2229. offset = i_size & (PAGE_CACHE_SIZE-1);
  2230. if (page->index == end_index && offset)
  2231. zero_user_segment(page, offset, PAGE_CACHE_SIZE);
  2232. SetPageUptodate(page);
  2233. } else {
  2234. ClearPageUptodate(page);
  2235. SetPageError(page);
  2236. }
  2237. unlock_page(page);
  2238. offset += len;
  2239. if (unlikely(!uptodate)) {
  2240. if (extent_len) {
  2241. endio_readpage_release_extent(tree,
  2242. extent_start,
  2243. extent_len, 1);
  2244. extent_start = 0;
  2245. extent_len = 0;
  2246. }
  2247. endio_readpage_release_extent(tree, start,
  2248. end - start + 1, 0);
  2249. } else if (!extent_len) {
  2250. extent_start = start;
  2251. extent_len = end + 1 - start;
  2252. } else if (extent_start + extent_len == start) {
  2253. extent_len += end + 1 - start;
  2254. } else {
  2255. endio_readpage_release_extent(tree, extent_start,
  2256. extent_len, uptodate);
  2257. extent_start = start;
  2258. extent_len = end + 1 - start;
  2259. }
  2260. } while (bvec <= bvec_end);
  2261. if (extent_len)
  2262. endio_readpage_release_extent(tree, extent_start, extent_len,
  2263. uptodate);
  2264. if (io_bio->end_io)
  2265. io_bio->end_io(io_bio, err);
  2266. bio_put(bio);
  2267. }
  2268. /*
  2269. * this allocates from the btrfs_bioset. We're returning a bio right now
  2270. * but you can call btrfs_io_bio for the appropriate container_of magic
  2271. */
  2272. struct bio *
  2273. btrfs_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs,
  2274. gfp_t gfp_flags)
  2275. {
  2276. struct btrfs_io_bio *btrfs_bio;
  2277. struct bio *bio;
  2278. bio = bio_alloc_bioset(gfp_flags, nr_vecs, btrfs_bioset);
  2279. if (bio == NULL && (current->flags & PF_MEMALLOC)) {
  2280. while (!bio && (nr_vecs /= 2)) {
  2281. bio = bio_alloc_bioset(gfp_flags,
  2282. nr_vecs, btrfs_bioset);
  2283. }
  2284. }
  2285. if (bio) {
  2286. bio->bi_size = 0;
  2287. bio->bi_bdev = bdev;
  2288. bio->bi_sector = first_sector;
  2289. btrfs_bio = btrfs_io_bio(bio);
  2290. btrfs_bio->csum = NULL;
  2291. btrfs_bio->csum_allocated = NULL;
  2292. btrfs_bio->end_io = NULL;
  2293. }
  2294. return bio;
  2295. }
  2296. struct bio *btrfs_bio_clone(struct bio *bio, gfp_t gfp_mask)
  2297. {
  2298. return bio_clone_bioset(bio, gfp_mask, btrfs_bioset);
  2299. }
  2300. /* this also allocates from the btrfs_bioset */
  2301. struct bio *btrfs_io_bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs)
  2302. {
  2303. struct btrfs_io_bio *btrfs_bio;
  2304. struct bio *bio;
  2305. bio = bio_alloc_bioset(gfp_mask, nr_iovecs, btrfs_bioset);
  2306. if (bio) {
  2307. btrfs_bio = btrfs_io_bio(bio);
  2308. btrfs_bio->csum = NULL;
  2309. btrfs_bio->csum_allocated = NULL;
  2310. btrfs_bio->end_io = NULL;
  2311. }
  2312. return bio;
  2313. }
  2314. static int __must_check submit_one_bio(int rw, struct bio *bio,
  2315. int mirror_num, unsigned long bio_flags)
  2316. {
  2317. int ret = 0;
  2318. struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
  2319. struct page *page = bvec->bv_page;
  2320. struct extent_io_tree *tree = bio->bi_private;
  2321. u64 start;
  2322. start = page_offset(page) + bvec->bv_offset;
  2323. bio->bi_private = NULL;
  2324. bio_get(bio);
  2325. if (tree->ops && tree->ops->submit_bio_hook)
  2326. ret = tree->ops->submit_bio_hook(page->mapping->host, rw, bio,
  2327. mirror_num, bio_flags, start);
  2328. else
  2329. btrfsic_submit_bio(rw, bio);
  2330. if (bio_flagged(bio, BIO_EOPNOTSUPP))
  2331. ret = -EOPNOTSUPP;
  2332. bio_put(bio);
  2333. return ret;
  2334. }
  2335. static int merge_bio(int rw, struct extent_io_tree *tree, struct page *page,
  2336. unsigned long offset, size_t size, struct bio *bio,
  2337. unsigned long bio_flags)
  2338. {
  2339. int ret = 0;
  2340. if (tree->ops && tree->ops->merge_bio_hook)
  2341. ret = tree->ops->merge_bio_hook(rw, page, offset, size, bio,
  2342. bio_flags);
  2343. BUG_ON(ret < 0);
  2344. return ret;
  2345. }
  2346. static int submit_extent_page(int rw, struct extent_io_tree *tree,
  2347. struct page *page, sector_t sector,
  2348. size_t size, unsigned long offset,
  2349. struct block_device *bdev,
  2350. struct bio **bio_ret,
  2351. unsigned long max_pages,
  2352. bio_end_io_t end_io_func,
  2353. int mirror_num,
  2354. unsigned long prev_bio_flags,
  2355. unsigned long bio_flags)
  2356. {
  2357. int ret = 0;
  2358. struct bio *bio;
  2359. int nr;
  2360. int contig = 0;
  2361. int this_compressed = bio_flags & EXTENT_BIO_COMPRESSED;
  2362. int old_compressed = prev_bio_flags & EXTENT_BIO_COMPRESSED;
  2363. size_t page_size = min_t(size_t, size, PAGE_CACHE_SIZE);
  2364. if (bio_ret && *bio_ret) {
  2365. bio = *bio_ret;
  2366. if (old_compressed)
  2367. contig = bio->bi_sector == sector;
  2368. else
  2369. contig = bio_end_sector(bio) == sector;
  2370. if (prev_bio_flags != bio_flags || !contig ||
  2371. merge_bio(rw, tree, page, offset, page_size, bio, bio_flags) ||
  2372. bio_add_page(bio, page, page_size, offset) < page_size) {
  2373. ret = submit_one_bio(rw, bio, mirror_num,
  2374. prev_bio_flags);
  2375. if (ret < 0)
  2376. return ret;
  2377. bio = NULL;
  2378. } else {
  2379. return 0;
  2380. }
  2381. }
  2382. if (this_compressed)
  2383. nr = BIO_MAX_PAGES;
  2384. else
  2385. nr = bio_get_nr_vecs(bdev);
  2386. bio = btrfs_bio_alloc(bdev, sector, nr, GFP_NOFS | __GFP_HIGH);
  2387. if (!bio)
  2388. return -ENOMEM;
  2389. bio_add_page(bio, page, page_size, offset);
  2390. bio->bi_end_io = end_io_func;
  2391. bio->bi_private = tree;
  2392. if (bio_ret)
  2393. *bio_ret = bio;
  2394. else
  2395. ret = submit_one_bio(rw, bio, mirror_num, bio_flags);
  2396. return ret;
  2397. }
  2398. static void attach_extent_buffer_page(struct extent_buffer *eb,
  2399. struct page *page)
  2400. {
  2401. if (!PagePrivate(page)) {
  2402. SetPagePrivate(page);
  2403. page_cache_get(page);
  2404. set_page_private(page, (unsigned long)eb);
  2405. } else {
  2406. WARN_ON(page->private != (unsigned long)eb);
  2407. }
  2408. }
  2409. void set_page_extent_mapped(struct page *page)
  2410. {
  2411. if (!PagePrivate(page)) {
  2412. SetPagePrivate(page);
  2413. page_cache_get(page);
  2414. set_page_private(page, EXTENT_PAGE_PRIVATE);
  2415. }
  2416. }
  2417. static struct extent_map *
  2418. __get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
  2419. u64 start, u64 len, get_extent_t *get_extent,
  2420. struct extent_map **em_cached)
  2421. {
  2422. struct extent_map *em;
  2423. if (em_cached && *em_cached) {
  2424. em = *em_cached;
  2425. if (em->in_tree && start >= em->start &&
  2426. start < extent_map_end(em)) {
  2427. atomic_inc(&em->refs);
  2428. return em;
  2429. }
  2430. free_extent_map(em);
  2431. *em_cached = NULL;
  2432. }
  2433. em = get_extent(inode, page, pg_offset, start, len, 0);
  2434. if (em_cached && !IS_ERR_OR_NULL(em)) {
  2435. BUG_ON(*em_cached);
  2436. atomic_inc(&em->refs);
  2437. *em_cached = em;
  2438. }
  2439. return em;
  2440. }
  2441. /*
  2442. * basic readpage implementation. Locked extent state structs are inserted
  2443. * into the tree that are removed when the IO is done (by the end_io
  2444. * handlers)
  2445. * XXX JDM: This needs looking at to ensure proper page locking
  2446. */
  2447. static int __do_readpage(struct extent_io_tree *tree,
  2448. struct page *page,
  2449. get_extent_t *get_extent,
  2450. struct extent_map **em_cached,
  2451. struct bio **bio, int mirror_num,
  2452. unsigned long *bio_flags, int rw)
  2453. {
  2454. struct inode *inode = page->mapping->host;
  2455. u64 start = page_offset(page);
  2456. u64 page_end = start + PAGE_CACHE_SIZE - 1;
  2457. u64 end;
  2458. u64 cur = start;
  2459. u64 extent_offset;
  2460. u64 last_byte = i_size_read(inode);
  2461. u64 block_start;
  2462. u64 cur_end;
  2463. sector_t sector;
  2464. struct extent_map *em;
  2465. struct block_device *bdev;
  2466. int ret;
  2467. int nr = 0;
  2468. int parent_locked = *bio_flags & EXTENT_BIO_PARENT_LOCKED;
  2469. size_t pg_offset = 0;
  2470. size_t iosize;
  2471. size_t disk_io_size;
  2472. size_t blocksize = inode->i_sb->s_blocksize;
  2473. unsigned long this_bio_flag = *bio_flags & EXTENT_BIO_PARENT_LOCKED;
  2474. set_page_extent_mapped(page);
  2475. end = page_end;
  2476. if (!PageUptodate(page)) {
  2477. if (cleancache_get_page(page) == 0) {
  2478. BUG_ON(blocksize != PAGE_SIZE);
  2479. unlock_extent(tree, start, end);
  2480. goto out;
  2481. }
  2482. }
  2483. if (page->index == last_byte >> PAGE_CACHE_SHIFT) {
  2484. char *userpage;
  2485. size_t zero_offset = last_byte & (PAGE_CACHE_SIZE - 1);
  2486. if (zero_offset) {
  2487. iosize = PAGE_CACHE_SIZE - zero_offset;
  2488. userpage = kmap_atomic(page);
  2489. memset(userpage + zero_offset, 0, iosize);
  2490. flush_dcache_page(page);
  2491. kunmap_atomic(userpage);
  2492. }
  2493. }
  2494. while (cur <= end) {
  2495. unsigned long pnr = (last_byte >> PAGE_CACHE_SHIFT) + 1;
  2496. if (cur >= last_byte) {
  2497. char *userpage;
  2498. struct extent_state *cached = NULL;
  2499. iosize = PAGE_CACHE_SIZE - pg_offset;
  2500. userpage = kmap_atomic(page);
  2501. memset(userpage + pg_offset, 0, iosize);
  2502. flush_dcache_page(page);
  2503. kunmap_atomic(userpage);
  2504. set_extent_uptodate(tree, cur, cur + iosize - 1,
  2505. &cached, GFP_NOFS);
  2506. if (!parent_locked)
  2507. unlock_extent_cached(tree, cur,
  2508. cur + iosize - 1,
  2509. &cached, GFP_NOFS);
  2510. break;
  2511. }
  2512. em = __get_extent_map(inode, page, pg_offset, cur,
  2513. end - cur + 1, get_extent, em_cached);
  2514. if (IS_ERR_OR_NULL(em)) {
  2515. SetPageError(page);
  2516. if (!parent_locked)
  2517. unlock_extent(tree, cur, end);
  2518. break;
  2519. }
  2520. extent_offset = cur - em->start;
  2521. BUG_ON(extent_map_end(em) <= cur);
  2522. BUG_ON(end < cur);
  2523. if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
  2524. this_bio_flag |= EXTENT_BIO_COMPRESSED;
  2525. extent_set_compress_type(&this_bio_flag,
  2526. em->compress_type);
  2527. }
  2528. iosize = min(extent_map_end(em) - cur, end - cur + 1);
  2529. cur_end = min(extent_map_end(em) - 1, end);
  2530. iosize = ALIGN(iosize, blocksize);
  2531. if (this_bio_flag & EXTENT_BIO_COMPRESSED) {
  2532. disk_io_size = em->block_len;
  2533. sector = em->block_start >> 9;
  2534. } else {
  2535. sector = (em->block_start + extent_offset) >> 9;
  2536. disk_io_size = iosize;
  2537. }
  2538. bdev = em->bdev;
  2539. block_start = em->block_start;
  2540. if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
  2541. block_start = EXTENT_MAP_HOLE;
  2542. free_extent_map(em);
  2543. em = NULL;
  2544. /* we've found a hole, just zero and go on */
  2545. if (block_start == EXTENT_MAP_HOLE) {
  2546. char *userpage;
  2547. struct extent_state *cached = NULL;
  2548. userpage = kmap_atomic(page);
  2549. memset(userpage + pg_offset, 0, iosize);
  2550. flush_dcache_page(page);
  2551. kunmap_atomic(userpage);
  2552. set_extent_uptodate(tree, cur, cur + iosize - 1,
  2553. &cached, GFP_NOFS);
  2554. unlock_extent_cached(tree, cur, cur + iosize - 1,
  2555. &cached, GFP_NOFS);
  2556. cur = cur + iosize;
  2557. pg_offset += iosize;
  2558. continue;
  2559. }
  2560. /* the get_extent function already copied into the page */
  2561. if (test_range_bit(tree, cur, cur_end,
  2562. EXTENT_UPTODATE, 1, NULL)) {
  2563. check_page_uptodate(tree, page);
  2564. if (!parent_locked)
  2565. unlock_extent(tree, cur, cur + iosize - 1);
  2566. cur = cur + iosize;
  2567. pg_offset += iosize;
  2568. continue;
  2569. }
  2570. /* we have an inline extent but it didn't get marked up
  2571. * to date. Error out
  2572. */
  2573. if (block_start == EXTENT_MAP_INLINE) {
  2574. SetPageError(page);
  2575. if (!parent_locked)
  2576. unlock_extent(tree, cur, cur + iosize - 1);
  2577. cur = cur + iosize;
  2578. pg_offset += iosize;
  2579. continue;
  2580. }
  2581. pnr -= page->index;
  2582. ret = submit_extent_page(rw, tree, page,
  2583. sector, disk_io_size, pg_offset,
  2584. bdev, bio, pnr,
  2585. end_bio_extent_readpage, mirror_num,
  2586. *bio_flags,
  2587. this_bio_flag);
  2588. if (!ret) {
  2589. nr++;
  2590. *bio_flags = this_bio_flag;
  2591. } else {
  2592. SetPageError(page);
  2593. if (!parent_locked)
  2594. unlock_extent(tree, cur, cur + iosize - 1);
  2595. }
  2596. cur = cur + iosize;
  2597. pg_offset += iosize;
  2598. }
  2599. out:
  2600. if (!nr) {
  2601. if (!PageError(page))
  2602. SetPageUptodate(page);
  2603. unlock_page(page);
  2604. }
  2605. return 0;
  2606. }
  2607. static inline void __do_contiguous_readpages(struct extent_io_tree *tree,
  2608. struct page *pages[], int nr_pages,
  2609. u64 start, u64 end,
  2610. get_extent_t *get_extent,
  2611. struct extent_map **em_cached,
  2612. struct bio **bio, int mirror_num,
  2613. unsigned long *bio_flags, int rw)
  2614. {
  2615. struct inode *inode;
  2616. struct btrfs_ordered_extent *ordered;
  2617. int index;
  2618. inode = pages[0]->mapping->host;
  2619. while (1) {
  2620. lock_extent(tree, start, end);
  2621. ordered = btrfs_lookup_ordered_range(inode, start,
  2622. end - start + 1);
  2623. if (!ordered)
  2624. break;
  2625. unlock_extent(tree, start, end);
  2626. btrfs_start_ordered_extent(inode, ordered, 1);
  2627. btrfs_put_ordered_extent(ordered);
  2628. }
  2629. for (index = 0; index < nr_pages; index++) {
  2630. __do_readpage(tree, pages[index], get_extent, em_cached, bio,
  2631. mirror_num, bio_flags, rw);
  2632. page_cache_release(pages[index]);
  2633. }
  2634. }
  2635. static void __extent_readpages(struct extent_io_tree *tree,
  2636. struct page *pages[],
  2637. int nr_pages, get_extent_t *get_extent,
  2638. struct extent_map **em_cached,
  2639. struct bio **bio, int mirror_num,
  2640. unsigned long *bio_flags, int rw)
  2641. {
  2642. u64 start = 0;
  2643. u64 end = 0;
  2644. u64 page_start;
  2645. int index;
  2646. int first_index = 0;
  2647. for (index = 0; index < nr_pages; index++) {
  2648. page_start = page_offset(pages[index]);
  2649. if (!end) {
  2650. start = page_start;
  2651. end = start + PAGE_CACHE_SIZE - 1;
  2652. first_index = index;
  2653. } else if (end + 1 == page_start) {
  2654. end += PAGE_CACHE_SIZE;
  2655. } else {
  2656. __do_contiguous_readpages(tree, &pages[first_index],
  2657. index - first_index, start,
  2658. end, get_extent, em_cached,
  2659. bio, mirror_num, bio_flags,
  2660. rw);
  2661. start = page_start;
  2662. end = start + PAGE_CACHE_SIZE - 1;
  2663. first_index = index;
  2664. }
  2665. }
  2666. if (end)
  2667. __do_contiguous_readpages(tree, &pages[first_index],
  2668. index - first_index, start,
  2669. end, get_extent, em_cached, bio,
  2670. mirror_num, bio_flags, rw);
  2671. }
  2672. static int __extent_read_full_page(struct extent_io_tree *tree,
  2673. struct page *page,
  2674. get_extent_t *get_extent,
  2675. struct bio **bio, int mirror_num,
  2676. unsigned long *bio_flags, int rw)
  2677. {
  2678. struct inode *inode = page->mapping->host;
  2679. struct btrfs_ordered_extent *ordered;
  2680. u64 start = page_offset(page);
  2681. u64 end = start + PAGE_CACHE_SIZE - 1;
  2682. int ret;
  2683. while (1) {
  2684. lock_extent(tree, start, end);
  2685. ordered = btrfs_lookup_ordered_extent(inode, start);
  2686. if (!ordered)
  2687. break;
  2688. unlock_extent(tree, start, end);
  2689. btrfs_start_ordered_extent(inode, ordered, 1);
  2690. btrfs_put_ordered_extent(ordered);
  2691. }
  2692. ret = __do_readpage(tree, page, get_extent, NULL, bio, mirror_num,
  2693. bio_flags, rw);
  2694. return ret;
  2695. }
  2696. int extent_read_full_page(struct extent_io_tree *tree, struct page *page,
  2697. get_extent_t *get_extent, int mirror_num)
  2698. {
  2699. struct bio *bio = NULL;
  2700. unsigned long bio_flags = 0;
  2701. int ret;
  2702. ret = __extent_read_full_page(tree, page, get_extent, &bio, mirror_num,
  2703. &bio_flags, READ);
  2704. if (bio)
  2705. ret = submit_one_bio(READ, bio, mirror_num, bio_flags);
  2706. return ret;
  2707. }
  2708. int extent_read_full_page_nolock(struct extent_io_tree *tree, struct page *page,
  2709. get_extent_t *get_extent, int mirror_num)
  2710. {
  2711. struct bio *bio = NULL;
  2712. unsigned long bio_flags = EXTENT_BIO_PARENT_LOCKED;
  2713. int ret;
  2714. ret = __do_readpage(tree, page, get_extent, NULL, &bio, mirror_num,
  2715. &bio_flags, READ);
  2716. if (bio)
  2717. ret = submit_one_bio(READ, bio, mirror_num, bio_flags);
  2718. return ret;
  2719. }
  2720. static noinline void update_nr_written(struct page *page,
  2721. struct writeback_control *wbc,
  2722. unsigned long nr_written)
  2723. {
  2724. wbc->nr_to_write -= nr_written;
  2725. if (wbc->range_cyclic || (wbc->nr_to_write > 0 &&
  2726. wbc->range_start == 0 && wbc->range_end == LLONG_MAX))
  2727. page->mapping->writeback_index = page->index + nr_written;
  2728. }
  2729. /*
  2730. * the writepage semantics are similar to regular writepage. extent
  2731. * records are inserted to lock ranges in the tree, and as dirty areas
  2732. * are found, they are marked writeback. Then the lock bits are removed
  2733. * and the end_io handler clears the writeback ranges
  2734. */
  2735. static int __extent_writepage(struct page *page, struct writeback_control *wbc,
  2736. void *data)
  2737. {
  2738. struct inode *inode = page->mapping->host;
  2739. struct extent_page_data *epd = data;
  2740. struct extent_io_tree *tree = epd->tree;
  2741. u64 start = page_offset(page);
  2742. u64 delalloc_start;
  2743. u64 page_end = start + PAGE_CACHE_SIZE - 1;
  2744. u64 end;
  2745. u64 cur = start;
  2746. u64 extent_offset;
  2747. u64 last_byte = i_size_read(inode);
  2748. u64 block_start;
  2749. u64 iosize;
  2750. sector_t sector;
  2751. struct extent_state *cached_state = NULL;
  2752. struct extent_map *em;
  2753. struct block_device *bdev;
  2754. int ret;
  2755. int nr = 0;
  2756. size_t pg_offset = 0;
  2757. size_t blocksize;
  2758. loff_t i_size = i_size_read(inode);
  2759. unsigned long end_index = i_size >> PAGE_CACHE_SHIFT;
  2760. u64 nr_delalloc;
  2761. u64 delalloc_end;
  2762. int page_started;
  2763. int compressed;
  2764. int write_flags;
  2765. unsigned long nr_written = 0;
  2766. bool fill_delalloc = true;
  2767. if (wbc->sync_mode == WB_SYNC_ALL)
  2768. write_flags = WRITE_SYNC;
  2769. else
  2770. write_flags = WRITE;
  2771. trace___extent_writepage(page, inode, wbc);
  2772. WARN_ON(!PageLocked(page));
  2773. ClearPageError(page);
  2774. pg_offset = i_size & (PAGE_CACHE_SIZE - 1);
  2775. if (page->index > end_index ||
  2776. (page->index == end_index && !pg_offset)) {
  2777. page->mapping->a_ops->invalidatepage(page, 0, PAGE_CACHE_SIZE);
  2778. unlock_page(page);
  2779. return 0;
  2780. }
  2781. if (page->index == end_index) {
  2782. char *userpage;
  2783. userpage = kmap_atomic(page);
  2784. memset(userpage + pg_offset, 0,
  2785. PAGE_CACHE_SIZE - pg_offset);
  2786. kunmap_atomic(userpage);
  2787. flush_dcache_page(page);
  2788. }
  2789. pg_offset = 0;
  2790. set_page_extent_mapped(page);
  2791. if (!tree->ops || !tree->ops->fill_delalloc)
  2792. fill_delalloc = false;
  2793. delalloc_start = start;
  2794. delalloc_end = 0;
  2795. page_started = 0;
  2796. if (!epd->extent_locked && fill_delalloc) {
  2797. u64 delalloc_to_write = 0;
  2798. /*
  2799. * make sure the wbc mapping index is at least updated
  2800. * to this page.
  2801. */
  2802. update_nr_written(page, wbc, 0);
  2803. while (delalloc_end < page_end) {
  2804. nr_delalloc = find_lock_delalloc_range(inode, tree,
  2805. page,
  2806. &delalloc_start,
  2807. &delalloc_end,
  2808. 128 * 1024 * 1024);
  2809. if (nr_delalloc == 0) {
  2810. delalloc_start = delalloc_end + 1;
  2811. continue;
  2812. }
  2813. ret = tree->ops->fill_delalloc(inode, page,
  2814. delalloc_start,
  2815. delalloc_end,
  2816. &page_started,
  2817. &nr_written);
  2818. /* File system has been set read-only */
  2819. if (ret) {
  2820. SetPageError(page);
  2821. goto done;
  2822. }
  2823. /*
  2824. * delalloc_end is already one less than the total
  2825. * length, so we don't subtract one from
  2826. * PAGE_CACHE_SIZE
  2827. */
  2828. delalloc_to_write += (delalloc_end - delalloc_start +
  2829. PAGE_CACHE_SIZE) >>
  2830. PAGE_CACHE_SHIFT;
  2831. delalloc_start = delalloc_end + 1;
  2832. }
  2833. if (wbc->nr_to_write < delalloc_to_write) {
  2834. int thresh = 8192;
  2835. if (delalloc_to_write < thresh * 2)
  2836. thresh = delalloc_to_write;
  2837. wbc->nr_to_write = min_t(u64, delalloc_to_write,
  2838. thresh);
  2839. }
  2840. /* did the fill delalloc function already unlock and start
  2841. * the IO?
  2842. */
  2843. if (page_started) {
  2844. ret = 0;
  2845. /*
  2846. * we've unlocked the page, so we can't update
  2847. * the mapping's writeback index, just update
  2848. * nr_to_write.
  2849. */
  2850. wbc->nr_to_write -= nr_written;
  2851. goto done_unlocked;
  2852. }
  2853. }
  2854. if (tree->ops && tree->ops->writepage_start_hook) {
  2855. ret = tree->ops->writepage_start_hook(page, start,
  2856. page_end);
  2857. if (ret) {
  2858. /* Fixup worker will requeue */
  2859. if (ret == -EBUSY)
  2860. wbc->pages_skipped++;
  2861. else
  2862. redirty_page_for_writepage(wbc, page);
  2863. update_nr_written(page, wbc, nr_written);
  2864. unlock_page(page);
  2865. ret = 0;
  2866. goto done_unlocked;
  2867. }
  2868. }
  2869. /*
  2870. * we don't want to touch the inode after unlocking the page,
  2871. * so we update the mapping writeback index now
  2872. */
  2873. update_nr_written(page, wbc, nr_written + 1);
  2874. end = page_end;
  2875. if (last_byte <= start) {
  2876. if (tree->ops && tree->ops->writepage_end_io_hook)
  2877. tree->ops->writepage_end_io_hook(page, start,
  2878. page_end, NULL, 1);
  2879. goto done;
  2880. }
  2881. blocksize = inode->i_sb->s_blocksize;
  2882. while (cur <= end) {
  2883. if (cur >= last_byte) {
  2884. if (tree->ops && tree->ops->writepage_end_io_hook)
  2885. tree->ops->writepage_end_io_hook(page, cur,
  2886. page_end, NULL, 1);
  2887. break;
  2888. }
  2889. em = epd->get_extent(inode, page, pg_offset, cur,
  2890. end - cur + 1, 1);
  2891. if (IS_ERR_OR_NULL(em)) {
  2892. SetPageError(page);
  2893. break;
  2894. }
  2895. extent_offset = cur - em->start;
  2896. BUG_ON(extent_map_end(em) <= cur);
  2897. BUG_ON(end < cur);
  2898. iosize = min(extent_map_end(em) - cur, end - cur + 1);
  2899. iosize = ALIGN(iosize, blocksize);
  2900. sector = (em->block_start + extent_offset) >> 9;
  2901. bdev = em->bdev;
  2902. block_start = em->block_start;
  2903. compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
  2904. free_extent_map(em);
  2905. em = NULL;
  2906. /*
  2907. * compressed and inline extents are written through other
  2908. * paths in the FS
  2909. */
  2910. if (compressed || block_start == EXTENT_MAP_HOLE ||
  2911. block_start == EXTENT_MAP_INLINE) {
  2912. /*
  2913. * end_io notification does not happen here for
  2914. * compressed extents
  2915. */
  2916. if (!compressed && tree->ops &&
  2917. tree->ops->writepage_end_io_hook)
  2918. tree->ops->writepage_end_io_hook(page, cur,
  2919. cur + iosize - 1,
  2920. NULL, 1);
  2921. else if (compressed) {
  2922. /* we don't want to end_page_writeback on
  2923. * a compressed extent. this happens
  2924. * elsewhere
  2925. */
  2926. nr++;
  2927. }
  2928. cur += iosize;
  2929. pg_offset += iosize;
  2930. continue;
  2931. }
  2932. /* leave this out until we have a page_mkwrite call */
  2933. if (0 && !test_range_bit(tree, cur, cur + iosize - 1,
  2934. EXTENT_DIRTY, 0, NULL)) {
  2935. cur = cur + iosize;
  2936. pg_offset += iosize;
  2937. continue;
  2938. }
  2939. if (tree->ops && tree->ops->writepage_io_hook) {
  2940. ret = tree->ops->writepage_io_hook(page, cur,
  2941. cur + iosize - 1);
  2942. } else {
  2943. ret = 0;
  2944. }
  2945. if (ret) {
  2946. SetPageError(page);
  2947. } else {
  2948. unsigned long max_nr = end_index + 1;
  2949. set_range_writeback(tree, cur, cur + iosize - 1);
  2950. if (!PageWriteback(page)) {
  2951. printk(KERN_ERR "btrfs warning page %lu not "
  2952. "writeback, cur %llu end %llu\n",
  2953. page->index, cur, end);
  2954. }
  2955. ret = submit_extent_page(write_flags, tree, page,
  2956. sector, iosize, pg_offset,
  2957. bdev, &epd->bio, max_nr,
  2958. end_bio_extent_writepage,
  2959. 0, 0, 0);
  2960. if (ret)
  2961. SetPageError(page);
  2962. }
  2963. cur = cur + iosize;
  2964. pg_offset += iosize;
  2965. nr++;
  2966. }
  2967. done:
  2968. if (nr == 0) {
  2969. /* make sure the mapping tag for page dirty gets cleared */
  2970. set_page_writeback(page);
  2971. end_page_writeback(page);
  2972. }
  2973. unlock_page(page);
  2974. done_unlocked:
  2975. /* drop our reference on any cached states */
  2976. free_extent_state(cached_state);
  2977. return 0;
  2978. }
  2979. static int eb_wait(void *word)
  2980. {
  2981. io_schedule();
  2982. return 0;
  2983. }
  2984. void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
  2985. {
  2986. wait_on_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK, eb_wait,
  2987. TASK_UNINTERRUPTIBLE);
  2988. }
  2989. static int lock_extent_buffer_for_io(struct extent_buffer *eb,
  2990. struct btrfs_fs_info *fs_info,
  2991. struct extent_page_data *epd)
  2992. {
  2993. unsigned long i, num_pages;
  2994. int flush = 0;
  2995. int ret = 0;
  2996. if (!btrfs_try_tree_write_lock(eb)) {
  2997. flush = 1;
  2998. flush_write_bio(epd);
  2999. btrfs_tree_lock(eb);
  3000. }
  3001. if (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
  3002. btrfs_tree_unlock(eb);
  3003. if (!epd->sync_io)
  3004. return 0;
  3005. if (!flush) {
  3006. flush_write_bio(epd);
  3007. flush = 1;
  3008. }
  3009. while (1) {
  3010. wait_on_extent_buffer_writeback(eb);
  3011. btrfs_tree_lock(eb);
  3012. if (!test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags))
  3013. break;
  3014. btrfs_tree_unlock(eb);
  3015. }
  3016. }
  3017. /*
  3018. * We need to do this to prevent races in people who check if the eb is
  3019. * under IO since we can end up having no IO bits set for a short period
  3020. * of time.
  3021. */
  3022. spin_lock(&eb->refs_lock);
  3023. if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
  3024. set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
  3025. spin_unlock(&eb->refs_lock);
  3026. btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
  3027. __percpu_counter_add(&fs_info->dirty_metadata_bytes,
  3028. -eb->len,
  3029. fs_info->dirty_metadata_batch);
  3030. ret = 1;
  3031. } else {
  3032. spin_unlock(&eb->refs_lock);
  3033. }
  3034. btrfs_tree_unlock(eb);
  3035. if (!ret)
  3036. return ret;
  3037. num_pages = num_extent_pages(eb->start, eb->len);
  3038. for (i = 0; i < num_pages; i++) {
  3039. struct page *p = extent_buffer_page(eb, i);
  3040. if (!trylock_page(p)) {
  3041. if (!flush) {
  3042. flush_write_bio(epd);
  3043. flush = 1;
  3044. }
  3045. lock_page(p);
  3046. }
  3047. }
  3048. return ret;
  3049. }
  3050. static void end_extent_buffer_writeback(struct extent_buffer *eb)
  3051. {
  3052. clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
  3053. smp_mb__after_clear_bit();
  3054. wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
  3055. }
  3056. static void end_bio_extent_buffer_writepage(struct bio *bio, int err)
  3057. {
  3058. int uptodate = err == 0;
  3059. struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
  3060. struct extent_buffer *eb;
  3061. int done;
  3062. do {
  3063. struct page *page = bvec->bv_page;
  3064. bvec--;
  3065. eb = (struct extent_buffer *)page->private;
  3066. BUG_ON(!eb);
  3067. done = atomic_dec_and_test(&eb->io_pages);
  3068. if (!uptodate || test_bit(EXTENT_BUFFER_IOERR, &eb->bflags)) {
  3069. set_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
  3070. ClearPageUptodate(page);
  3071. SetPageError(page);
  3072. }
  3073. end_page_writeback(page);
  3074. if (!done)
  3075. continue;
  3076. end_extent_buffer_writeback(eb);
  3077. } while (bvec >= bio->bi_io_vec);
  3078. bio_put(bio);
  3079. }
  3080. static int write_one_eb(struct extent_buffer *eb,
  3081. struct btrfs_fs_info *fs_info,
  3082. struct writeback_control *wbc,
  3083. struct extent_page_data *epd)
  3084. {
  3085. struct block_device *bdev = fs_info->fs_devices->latest_bdev;
  3086. u64 offset = eb->start;
  3087. unsigned long i, num_pages;
  3088. unsigned long bio_flags = 0;
  3089. int rw = (epd->sync_io ? WRITE_SYNC : WRITE) | REQ_META;
  3090. int ret = 0;
  3091. clear_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
  3092. num_pages = num_extent_pages(eb->start, eb->len);
  3093. atomic_set(&eb->io_pages, num_pages);
  3094. if (btrfs_header_owner(eb) == BTRFS_TREE_LOG_OBJECTID)
  3095. bio_flags = EXTENT_BIO_TREE_LOG;
  3096. for (i = 0; i < num_pages; i++) {
  3097. struct page *p = extent_buffer_page(eb, i);
  3098. clear_page_dirty_for_io(p);
  3099. set_page_writeback(p);
  3100. ret = submit_extent_page(rw, eb->tree, p, offset >> 9,
  3101. PAGE_CACHE_SIZE, 0, bdev, &epd->bio,
  3102. -1, end_bio_extent_buffer_writepage,
  3103. 0, epd->bio_flags, bio_flags);
  3104. epd->bio_flags = bio_flags;
  3105. if (ret) {
  3106. set_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
  3107. SetPageError(p);
  3108. if (atomic_sub_and_test(num_pages - i, &eb->io_pages))
  3109. end_extent_buffer_writeback(eb);
  3110. ret = -EIO;
  3111. break;
  3112. }
  3113. offset += PAGE_CACHE_SIZE;
  3114. update_nr_written(p, wbc, 1);
  3115. unlock_page(p);
  3116. }
  3117. if (unlikely(ret)) {
  3118. for (; i < num_pages; i++) {
  3119. struct page *p = extent_buffer_page(eb, i);
  3120. unlock_page(p);
  3121. }
  3122. }
  3123. return ret;
  3124. }
  3125. int btree_write_cache_pages(struct address_space *mapping,
  3126. struct writeback_control *wbc)
  3127. {
  3128. struct extent_io_tree *tree = &BTRFS_I(mapping->host)->io_tree;
  3129. struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info;
  3130. struct extent_buffer *eb, *prev_eb = NULL;
  3131. struct extent_page_data epd = {
  3132. .bio = NULL,
  3133. .tree = tree,
  3134. .extent_locked = 0,
  3135. .sync_io = wbc->sync_mode == WB_SYNC_ALL,
  3136. .bio_flags = 0,
  3137. };
  3138. int ret = 0;
  3139. int done = 0;
  3140. int nr_to_write_done = 0;
  3141. struct pagevec pvec;
  3142. int nr_pages;
  3143. pgoff_t index;
  3144. pgoff_t end; /* Inclusive */
  3145. int scanned = 0;
  3146. int tag;
  3147. pagevec_init(&pvec, 0);
  3148. if (wbc->range_cyclic) {
  3149. index = mapping->writeback_index; /* Start from prev offset */
  3150. end = -1;
  3151. } else {
  3152. index = wbc->range_start >> PAGE_CACHE_SHIFT;
  3153. end = wbc->range_end >> PAGE_CACHE_SHIFT;
  3154. scanned = 1;
  3155. }
  3156. if (wbc->sync_mode == WB_SYNC_ALL)
  3157. tag = PAGECACHE_TAG_TOWRITE;
  3158. else
  3159. tag = PAGECACHE_TAG_DIRTY;
  3160. retry:
  3161. if (wbc->sync_mode == WB_SYNC_ALL)
  3162. tag_pages_for_writeback(mapping, index, end);
  3163. while (!done && !nr_to_write_done && (index <= end) &&
  3164. (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
  3165. min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
  3166. unsigned i;
  3167. scanned = 1;
  3168. for (i = 0; i < nr_pages; i++) {
  3169. struct page *page = pvec.pages[i];
  3170. if (!PagePrivate(page))
  3171. continue;
  3172. if (!wbc->range_cyclic && page->index > end) {
  3173. done = 1;
  3174. break;
  3175. }
  3176. spin_lock(&mapping->private_lock);
  3177. if (!PagePrivate(page)) {
  3178. spin_unlock(&mapping->private_lock);
  3179. continue;
  3180. }
  3181. eb = (struct extent_buffer *)page->private;
  3182. /*
  3183. * Shouldn't happen and normally this would be a BUG_ON
  3184. * but no sense in crashing the users box for something
  3185. * we can survive anyway.
  3186. */
  3187. if (WARN_ON(!eb)) {
  3188. spin_unlock(&mapping->private_lock);
  3189. continue;
  3190. }
  3191. if (eb == prev_eb) {
  3192. spin_unlock(&mapping->private_lock);
  3193. continue;
  3194. }
  3195. ret = atomic_inc_not_zero(&eb->refs);
  3196. spin_unlock(&mapping->private_lock);
  3197. if (!ret)
  3198. continue;
  3199. prev_eb = eb;
  3200. ret = lock_extent_buffer_for_io(eb, fs_info, &epd);
  3201. if (!ret) {
  3202. free_extent_buffer(eb);
  3203. continue;
  3204. }
  3205. ret = write_one_eb(eb, fs_info, wbc, &epd);
  3206. if (ret) {
  3207. done = 1;
  3208. free_extent_buffer(eb);
  3209. break;
  3210. }
  3211. free_extent_buffer(eb);
  3212. /*
  3213. * the filesystem may choose to bump up nr_to_write.
  3214. * We have to make sure to honor the new nr_to_write
  3215. * at any time
  3216. */
  3217. nr_to_write_done = wbc->nr_to_write <= 0;
  3218. }
  3219. pagevec_release(&pvec);
  3220. cond_resched();
  3221. }
  3222. if (!scanned && !done) {
  3223. /*
  3224. * We hit the last page and there is more work to be done: wrap
  3225. * back to the start of the file
  3226. */
  3227. scanned = 1;
  3228. index = 0;
  3229. goto retry;
  3230. }
  3231. flush_write_bio(&epd);
  3232. return ret;
  3233. }
  3234. /**
  3235. * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
  3236. * @mapping: address space structure to write
  3237. * @wbc: subtract the number of written pages from *@wbc->nr_to_write
  3238. * @writepage: function called for each page
  3239. * @data: data passed to writepage function
  3240. *
  3241. * If a page is already under I/O, write_cache_pages() skips it, even
  3242. * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
  3243. * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
  3244. * and msync() need to guarantee that all the data which was dirty at the time
  3245. * the call was made get new I/O started against them. If wbc->sync_mode is
  3246. * WB_SYNC_ALL then we were called for data integrity and we must wait for
  3247. * existing IO to complete.
  3248. */
  3249. static int extent_write_cache_pages(struct extent_io_tree *tree,
  3250. struct address_space *mapping,
  3251. struct writeback_control *wbc,
  3252. writepage_t writepage, void *data,
  3253. void (*flush_fn)(void *))
  3254. {
  3255. struct inode *inode = mapping->host;
  3256. int ret = 0;
  3257. int done = 0;
  3258. int nr_to_write_done = 0;
  3259. struct pagevec pvec;
  3260. int nr_pages;
  3261. pgoff_t index;
  3262. pgoff_t end; /* Inclusive */
  3263. int scanned = 0;
  3264. int tag;
  3265. /*
  3266. * We have to hold onto the inode so that ordered extents can do their
  3267. * work when the IO finishes. The alternative to this is failing to add
  3268. * an ordered extent if the igrab() fails there and that is a huge pain
  3269. * to deal with, so instead just hold onto the inode throughout the
  3270. * writepages operation. If it fails here we are freeing up the inode
  3271. * anyway and we'd rather not waste our time writing out stuff that is
  3272. * going to be truncated anyway.
  3273. */
  3274. if (!igrab(inode))
  3275. return 0;
  3276. pagevec_init(&pvec, 0);
  3277. if (wbc->range_cyclic) {
  3278. index = mapping->writeback_index; /* Start from prev offset */
  3279. end = -1;
  3280. } else {
  3281. index = wbc->range_start >> PAGE_CACHE_SHIFT;
  3282. end = wbc->range_end >> PAGE_CACHE_SHIFT;
  3283. scanned = 1;
  3284. }
  3285. if (wbc->sync_mode == WB_SYNC_ALL)
  3286. tag = PAGECACHE_TAG_TOWRITE;
  3287. else
  3288. tag = PAGECACHE_TAG_DIRTY;
  3289. retry:
  3290. if (wbc->sync_mode == WB_SYNC_ALL)
  3291. tag_pages_for_writeback(mapping, index, end);
  3292. while (!done && !nr_to_write_done && (index <= end) &&
  3293. (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
  3294. min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
  3295. unsigned i;
  3296. scanned = 1;
  3297. for (i = 0; i < nr_pages; i++) {
  3298. struct page *page = pvec.pages[i];
  3299. /*
  3300. * At this point we hold neither mapping->tree_lock nor
  3301. * lock on the page itself: the page may be truncated or
  3302. * invalidated (changing page->mapping to NULL), or even
  3303. * swizzled back from swapper_space to tmpfs file
  3304. * mapping
  3305. */
  3306. if (!trylock_page(page)) {
  3307. flush_fn(data);
  3308. lock_page(page);
  3309. }
  3310. if (unlikely(page->mapping != mapping)) {
  3311. unlock_page(page);
  3312. continue;
  3313. }
  3314. if (!wbc->range_cyclic && page->index > end) {
  3315. done = 1;
  3316. unlock_page(page);
  3317. continue;
  3318. }
  3319. if (wbc->sync_mode != WB_SYNC_NONE) {
  3320. if (PageWriteback(page))
  3321. flush_fn(data);
  3322. wait_on_page_writeback(page);
  3323. }
  3324. if (PageWriteback(page) ||
  3325. !clear_page_dirty_for_io(page)) {
  3326. unlock_page(page);
  3327. continue;
  3328. }
  3329. ret = (*writepage)(page, wbc, data);
  3330. if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) {
  3331. unlock_page(page);
  3332. ret = 0;
  3333. }
  3334. if (ret)
  3335. done = 1;
  3336. /*
  3337. * the filesystem may choose to bump up nr_to_write.
  3338. * We have to make sure to honor the new nr_to_write
  3339. * at any time
  3340. */
  3341. nr_to_write_done = wbc->nr_to_write <= 0;
  3342. }
  3343. pagevec_release(&pvec);
  3344. cond_resched();
  3345. }
  3346. if (!scanned && !done) {
  3347. /*
  3348. * We hit the last page and there is more work to be done: wrap
  3349. * back to the start of the file
  3350. */
  3351. scanned = 1;
  3352. index = 0;
  3353. goto retry;
  3354. }
  3355. btrfs_add_delayed_iput(inode);
  3356. return ret;
  3357. }
  3358. static void flush_epd_write_bio(struct extent_page_data *epd)
  3359. {
  3360. if (epd->bio) {
  3361. int rw = WRITE;
  3362. int ret;
  3363. if (epd->sync_io)
  3364. rw = WRITE_SYNC;
  3365. ret = submit_one_bio(rw, epd->bio, 0, epd->bio_flags);
  3366. BUG_ON(ret < 0); /* -ENOMEM */
  3367. epd->bio = NULL;
  3368. }
  3369. }
  3370. static noinline void flush_write_bio(void *data)
  3371. {
  3372. struct extent_page_data *epd = data;
  3373. flush_epd_write_bio(epd);
  3374. }
  3375. int extent_write_full_page(struct extent_io_tree *tree, struct page *page,
  3376. get_extent_t *get_extent,
  3377. struct writeback_control *wbc)
  3378. {
  3379. int ret;
  3380. struct extent_page_data epd = {
  3381. .bio = NULL,
  3382. .tree = tree,
  3383. .get_extent = get_extent,
  3384. .extent_locked = 0,
  3385. .sync_io = wbc->sync_mode == WB_SYNC_ALL,
  3386. .bio_flags = 0,
  3387. };
  3388. ret = __extent_writepage(page, wbc, &epd);
  3389. flush_epd_write_bio(&epd);
  3390. return ret;
  3391. }
  3392. int extent_write_locked_range(struct extent_io_tree *tree, struct inode *inode,
  3393. u64 start, u64 end, get_extent_t *get_extent,
  3394. int mode)
  3395. {
  3396. int ret = 0;
  3397. struct address_space *mapping = inode->i_mapping;
  3398. struct page *page;
  3399. unsigned long nr_pages = (end - start + PAGE_CACHE_SIZE) >>
  3400. PAGE_CACHE_SHIFT;
  3401. struct extent_page_data epd = {
  3402. .bio = NULL,
  3403. .tree = tree,
  3404. .get_extent = get_extent,
  3405. .extent_locked = 1,
  3406. .sync_io = mode == WB_SYNC_ALL,
  3407. .bio_flags = 0,
  3408. };
  3409. struct writeback_control wbc_writepages = {
  3410. .sync_mode = mode,
  3411. .nr_to_write = nr_pages * 2,
  3412. .range_start = start,
  3413. .range_end = end + 1,
  3414. };
  3415. while (start <= end) {
  3416. page = find_get_page(mapping, start >> PAGE_CACHE_SHIFT);
  3417. if (clear_page_dirty_for_io(page))
  3418. ret = __extent_writepage(page, &wbc_writepages, &epd);
  3419. else {
  3420. if (tree->ops && tree->ops->writepage_end_io_hook)
  3421. tree->ops->writepage_end_io_hook(page, start,
  3422. start + PAGE_CACHE_SIZE - 1,
  3423. NULL, 1);
  3424. unlock_page(page);
  3425. }
  3426. page_cache_release(page);
  3427. start += PAGE_CACHE_SIZE;
  3428. }
  3429. flush_epd_write_bio(&epd);
  3430. return ret;
  3431. }
  3432. int extent_writepages(struct extent_io_tree *tree,
  3433. struct address_space *mapping,
  3434. get_extent_t *get_extent,
  3435. struct writeback_control *wbc)
  3436. {
  3437. int ret = 0;
  3438. struct extent_page_data epd = {
  3439. .bio = NULL,
  3440. .tree = tree,
  3441. .get_extent = get_extent,
  3442. .extent_locked = 0,
  3443. .sync_io = wbc->sync_mode == WB_SYNC_ALL,
  3444. .bio_flags = 0,
  3445. };
  3446. ret = extent_write_cache_pages(tree, mapping, wbc,
  3447. __extent_writepage, &epd,
  3448. flush_write_bio);
  3449. flush_epd_write_bio(&epd);
  3450. return ret;
  3451. }
  3452. int extent_readpages(struct extent_io_tree *tree,
  3453. struct address_space *mapping,
  3454. struct list_head *pages, unsigned nr_pages,
  3455. get_extent_t get_extent)
  3456. {
  3457. struct bio *bio = NULL;
  3458. unsigned page_idx;
  3459. unsigned long bio_flags = 0;
  3460. struct page *pagepool[16];
  3461. struct page *page;
  3462. struct extent_map *em_cached = NULL;
  3463. int nr = 0;
  3464. for (page_idx = 0; page_idx < nr_pages; page_idx++) {
  3465. page = list_entry(pages->prev, struct page, lru);
  3466. prefetchw(&page->flags);
  3467. list_del(&page->lru);
  3468. if (add_to_page_cache_lru(page, mapping,
  3469. page->index, GFP_NOFS)) {
  3470. page_cache_release(page);
  3471. continue;
  3472. }
  3473. pagepool[nr++] = page;
  3474. if (nr < ARRAY_SIZE(pagepool))
  3475. continue;
  3476. __extent_readpages(tree, pagepool, nr, get_extent, &em_cached,
  3477. &bio, 0, &bio_flags, READ);
  3478. nr = 0;
  3479. }
  3480. if (nr)
  3481. __extent_readpages(tree, pagepool, nr, get_extent, &em_cached,
  3482. &bio, 0, &bio_flags, READ);
  3483. if (em_cached)
  3484. free_extent_map(em_cached);
  3485. BUG_ON(!list_empty(pages));
  3486. if (bio)
  3487. return submit_one_bio(READ, bio, 0, bio_flags);
  3488. return 0;
  3489. }
  3490. /*
  3491. * basic invalidatepage code, this waits on any locked or writeback
  3492. * ranges corresponding to the page, and then deletes any extent state
  3493. * records from the tree
  3494. */
  3495. int extent_invalidatepage(struct extent_io_tree *tree,
  3496. struct page *page, unsigned long offset)
  3497. {
  3498. struct extent_state *cached_state = NULL;
  3499. u64 start = page_offset(page);
  3500. u64 end = start + PAGE_CACHE_SIZE - 1;
  3501. size_t blocksize = page->mapping->host->i_sb->s_blocksize;
  3502. start += ALIGN(offset, blocksize);
  3503. if (start > end)
  3504. return 0;
  3505. lock_extent_bits(tree, start, end, 0, &cached_state);
  3506. wait_on_page_writeback(page);
  3507. clear_extent_bit(tree, start, end,
  3508. EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC |
  3509. EXTENT_DO_ACCOUNTING,
  3510. 1, 1, &cached_state, GFP_NOFS);
  3511. return 0;
  3512. }
  3513. /*
  3514. * a helper for releasepage, this tests for areas of the page that
  3515. * are locked or under IO and drops the related state bits if it is safe
  3516. * to drop the page.
  3517. */
  3518. static int try_release_extent_state(struct extent_map_tree *map,
  3519. struct extent_io_tree *tree,
  3520. struct page *page, gfp_t mask)
  3521. {
  3522. u64 start = page_offset(page);
  3523. u64 end = start + PAGE_CACHE_SIZE - 1;
  3524. int ret = 1;
  3525. if (test_range_bit(tree, start, end,
  3526. EXTENT_IOBITS, 0, NULL))
  3527. ret = 0;
  3528. else {
  3529. if ((mask & GFP_NOFS) == GFP_NOFS)
  3530. mask = GFP_NOFS;
  3531. /*
  3532. * at this point we can safely clear everything except the
  3533. * locked bit and the nodatasum bit
  3534. */
  3535. ret = clear_extent_bit(tree, start, end,
  3536. ~(EXTENT_LOCKED | EXTENT_NODATASUM),
  3537. 0, 0, NULL, mask);
  3538. /* if clear_extent_bit failed for enomem reasons,
  3539. * we can't allow the release to continue.
  3540. */
  3541. if (ret < 0)
  3542. ret = 0;
  3543. else
  3544. ret = 1;
  3545. }
  3546. return ret;
  3547. }
  3548. /*
  3549. * a helper for releasepage. As long as there are no locked extents
  3550. * in the range corresponding to the page, both state records and extent
  3551. * map records are removed
  3552. */
  3553. int try_release_extent_mapping(struct extent_map_tree *map,
  3554. struct extent_io_tree *tree, struct page *page,
  3555. gfp_t mask)
  3556. {
  3557. struct extent_map *em;
  3558. u64 start = page_offset(page);
  3559. u64 end = start + PAGE_CACHE_SIZE - 1;
  3560. if ((mask & __GFP_WAIT) &&
  3561. page->mapping->host->i_size > 16 * 1024 * 1024) {
  3562. u64 len;
  3563. while (start <= end) {
  3564. len = end - start + 1;
  3565. write_lock(&map->lock);
  3566. em = lookup_extent_mapping(map, start, len);
  3567. if (!em) {
  3568. write_unlock(&map->lock);
  3569. break;
  3570. }
  3571. if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
  3572. em->start != start) {
  3573. write_unlock(&map->lock);
  3574. free_extent_map(em);
  3575. break;
  3576. }
  3577. if (!test_range_bit(tree, em->start,
  3578. extent_map_end(em) - 1,
  3579. EXTENT_LOCKED | EXTENT_WRITEBACK,
  3580. 0, NULL)) {
  3581. remove_extent_mapping(map, em);
  3582. /* once for the rb tree */
  3583. free_extent_map(em);
  3584. }
  3585. start = extent_map_end(em);
  3586. write_unlock(&map->lock);
  3587. /* once for us */
  3588. free_extent_map(em);
  3589. }
  3590. }
  3591. return try_release_extent_state(map, tree, page, mask);
  3592. }
  3593. /*
  3594. * helper function for fiemap, which doesn't want to see any holes.
  3595. * This maps until we find something past 'last'
  3596. */
  3597. static struct extent_map *get_extent_skip_holes(struct inode *inode,
  3598. u64 offset,
  3599. u64 last,
  3600. get_extent_t *get_extent)
  3601. {
  3602. u64 sectorsize = BTRFS_I(inode)->root->sectorsize;
  3603. struct extent_map *em;
  3604. u64 len;
  3605. if (offset >= last)
  3606. return NULL;
  3607. while (1) {
  3608. len = last - offset;
  3609. if (len == 0)
  3610. break;
  3611. len = ALIGN(len, sectorsize);
  3612. em = get_extent(inode, NULL, 0, offset, len, 0);
  3613. if (IS_ERR_OR_NULL(em))
  3614. return em;
  3615. /* if this isn't a hole return it */
  3616. if (!test_bit(EXTENT_FLAG_VACANCY, &em->flags) &&
  3617. em->block_start != EXTENT_MAP_HOLE) {
  3618. return em;
  3619. }
  3620. /* this is a hole, advance to the next extent */
  3621. offset = extent_map_end(em);
  3622. free_extent_map(em);
  3623. if (offset >= last)
  3624. break;
  3625. }
  3626. return NULL;
  3627. }
  3628. static noinline int count_ext_ref(u64 inum, u64 offset, u64 root_id, void *ctx)
  3629. {
  3630. unsigned long cnt = *((unsigned long *)ctx);
  3631. cnt++;
  3632. *((unsigned long *)ctx) = cnt;
  3633. /* Now we're sure that the extent is shared. */
  3634. if (cnt > 1)
  3635. return 1;
  3636. return 0;
  3637. }
  3638. int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
  3639. __u64 start, __u64 len, get_extent_t *get_extent)
  3640. {
  3641. int ret = 0;
  3642. u64 off = start;
  3643. u64 max = start + len;
  3644. u32 flags = 0;
  3645. u32 found_type;
  3646. u64 last;
  3647. u64 last_for_get_extent = 0;
  3648. u64 disko = 0;
  3649. u64 isize = i_size_read(inode);
  3650. struct btrfs_key found_key;
  3651. struct extent_map *em = NULL;
  3652. struct extent_state *cached_state = NULL;
  3653. struct btrfs_path *path;
  3654. struct btrfs_file_extent_item *item;
  3655. int end = 0;
  3656. u64 em_start = 0;
  3657. u64 em_len = 0;
  3658. u64 em_end = 0;
  3659. unsigned long emflags;
  3660. if (len == 0)
  3661. return -EINVAL;
  3662. path = btrfs_alloc_path();
  3663. if (!path)
  3664. return -ENOMEM;
  3665. path->leave_spinning = 1;
  3666. start = ALIGN(start, BTRFS_I(inode)->root->sectorsize);
  3667. len = ALIGN(len, BTRFS_I(inode)->root->sectorsize);
  3668. /*
  3669. * lookup the last file extent. We're not using i_size here
  3670. * because there might be preallocation past i_size
  3671. */
  3672. ret = btrfs_lookup_file_extent(NULL, BTRFS_I(inode)->root,
  3673. path, btrfs_ino(inode), -1, 0);
  3674. if (ret < 0) {
  3675. btrfs_free_path(path);
  3676. return ret;
  3677. }
  3678. WARN_ON(!ret);
  3679. path->slots[0]--;
  3680. item = btrfs_item_ptr(path->nodes[0], path->slots[0],
  3681. struct btrfs_file_extent_item);
  3682. btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]);
  3683. found_type = btrfs_key_type(&found_key);
  3684. /* No extents, but there might be delalloc bits */
  3685. if (found_key.objectid != btrfs_ino(inode) ||
  3686. found_type != BTRFS_EXTENT_DATA_KEY) {
  3687. /* have to trust i_size as the end */
  3688. last = (u64)-1;
  3689. last_for_get_extent = isize;
  3690. } else {
  3691. /*
  3692. * remember the start of the last extent. There are a
  3693. * bunch of different factors that go into the length of the
  3694. * extent, so its much less complex to remember where it started
  3695. */
  3696. last = found_key.offset;
  3697. last_for_get_extent = last + 1;
  3698. }
  3699. btrfs_release_path(path);
  3700. /*
  3701. * we might have some extents allocated but more delalloc past those
  3702. * extents. so, we trust isize unless the start of the last extent is
  3703. * beyond isize
  3704. */
  3705. if (last < isize) {
  3706. last = (u64)-1;
  3707. last_for_get_extent = isize;
  3708. }
  3709. lock_extent_bits(&BTRFS_I(inode)->io_tree, start, start + len - 1, 0,
  3710. &cached_state);
  3711. em = get_extent_skip_holes(inode, start, last_for_get_extent,
  3712. get_extent);
  3713. if (!em)
  3714. goto out;
  3715. if (IS_ERR(em)) {
  3716. ret = PTR_ERR(em);
  3717. goto out;
  3718. }
  3719. while (!end) {
  3720. u64 offset_in_extent = 0;
  3721. /* break if the extent we found is outside the range */
  3722. if (em->start >= max || extent_map_end(em) < off)
  3723. break;
  3724. /*
  3725. * get_extent may return an extent that starts before our
  3726. * requested range. We have to make sure the ranges
  3727. * we return to fiemap always move forward and don't
  3728. * overlap, so adjust the offsets here
  3729. */
  3730. em_start = max(em->start, off);
  3731. /*
  3732. * record the offset from the start of the extent
  3733. * for adjusting the disk offset below. Only do this if the
  3734. * extent isn't compressed since our in ram offset may be past
  3735. * what we have actually allocated on disk.
  3736. */
  3737. if (!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
  3738. offset_in_extent = em_start - em->start;
  3739. em_end = extent_map_end(em);
  3740. em_len = em_end - em_start;
  3741. emflags = em->flags;
  3742. disko = 0;
  3743. flags = 0;
  3744. /*
  3745. * bump off for our next call to get_extent
  3746. */
  3747. off = extent_map_end(em);
  3748. if (off >= max)
  3749. end = 1;
  3750. if (em->block_start == EXTENT_MAP_LAST_BYTE) {
  3751. end = 1;
  3752. flags |= FIEMAP_EXTENT_LAST;
  3753. } else if (em->block_start == EXTENT_MAP_INLINE) {
  3754. flags |= (FIEMAP_EXTENT_DATA_INLINE |
  3755. FIEMAP_EXTENT_NOT_ALIGNED);
  3756. } else if (em->block_start == EXTENT_MAP_DELALLOC) {
  3757. flags |= (FIEMAP_EXTENT_DELALLOC |
  3758. FIEMAP_EXTENT_UNKNOWN);
  3759. } else {
  3760. unsigned long ref_cnt = 0;
  3761. disko = em->block_start + offset_in_extent;
  3762. /*
  3763. * As btrfs supports shared space, this information
  3764. * can be exported to userspace tools via
  3765. * flag FIEMAP_EXTENT_SHARED.
  3766. */
  3767. ret = iterate_inodes_from_logical(
  3768. em->block_start,
  3769. BTRFS_I(inode)->root->fs_info,
  3770. path, count_ext_ref, &ref_cnt);
  3771. if (ret < 0 && ret != -ENOENT)
  3772. goto out_free;
  3773. if (ref_cnt > 1)
  3774. flags |= FIEMAP_EXTENT_SHARED;
  3775. }
  3776. if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
  3777. flags |= FIEMAP_EXTENT_ENCODED;
  3778. free_extent_map(em);
  3779. em = NULL;
  3780. if ((em_start >= last) || em_len == (u64)-1 ||
  3781. (last == (u64)-1 && isize <= em_end)) {
  3782. flags |= FIEMAP_EXTENT_LAST;
  3783. end = 1;
  3784. }
  3785. /* now scan forward to see if this is really the last extent. */
  3786. em = get_extent_skip_holes(inode, off, last_for_get_extent,
  3787. get_extent);
  3788. if (IS_ERR(em)) {
  3789. ret = PTR_ERR(em);
  3790. goto out;
  3791. }
  3792. if (!em) {
  3793. flags |= FIEMAP_EXTENT_LAST;
  3794. end = 1;
  3795. }
  3796. ret = fiemap_fill_next_extent(fieinfo, em_start, disko,
  3797. em_len, flags);
  3798. if (ret)
  3799. goto out_free;
  3800. }
  3801. out_free:
  3802. free_extent_map(em);
  3803. out:
  3804. btrfs_free_path(path);
  3805. unlock_extent_cached(&BTRFS_I(inode)->io_tree, start, start + len - 1,
  3806. &cached_state, GFP_NOFS);
  3807. return ret;
  3808. }
  3809. static void __free_extent_buffer(struct extent_buffer *eb)
  3810. {
  3811. btrfs_leak_debug_del(&eb->leak_list);
  3812. kmem_cache_free(extent_buffer_cache, eb);
  3813. }
  3814. static int extent_buffer_under_io(struct extent_buffer *eb)
  3815. {
  3816. return (atomic_read(&eb->io_pages) ||
  3817. test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
  3818. test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
  3819. }
  3820. /*
  3821. * Helper for releasing extent buffer page.
  3822. */
  3823. static void btrfs_release_extent_buffer_page(struct extent_buffer *eb,
  3824. unsigned long start_idx)
  3825. {
  3826. unsigned long index;
  3827. unsigned long num_pages;
  3828. struct page *page;
  3829. int mapped = !test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
  3830. BUG_ON(extent_buffer_under_io(eb));
  3831. num_pages = num_extent_pages(eb->start, eb->len);
  3832. index = start_idx + num_pages;
  3833. if (start_idx >= index)
  3834. return;
  3835. do {
  3836. index--;
  3837. page = extent_buffer_page(eb, index);
  3838. if (page && mapped) {
  3839. spin_lock(&page->mapping->private_lock);
  3840. /*
  3841. * We do this since we'll remove the pages after we've
  3842. * removed the eb from the radix tree, so we could race
  3843. * and have this page now attached to the new eb. So
  3844. * only clear page_private if it's still connected to
  3845. * this eb.
  3846. */
  3847. if (PagePrivate(page) &&
  3848. page->private == (unsigned long)eb) {
  3849. BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
  3850. BUG_ON(PageDirty(page));
  3851. BUG_ON(PageWriteback(page));
  3852. /*
  3853. * We need to make sure we haven't be attached
  3854. * to a new eb.
  3855. */
  3856. ClearPagePrivate(page);
  3857. set_page_private(page, 0);
  3858. /* One for the page private */
  3859. page_cache_release(page);
  3860. }
  3861. spin_unlock(&page->mapping->private_lock);
  3862. }
  3863. if (page) {
  3864. /* One for when we alloced the page */
  3865. page_cache_release(page);
  3866. }
  3867. } while (index != start_idx);
  3868. }
  3869. /*
  3870. * Helper for releasing the extent buffer.
  3871. */
  3872. static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
  3873. {
  3874. btrfs_release_extent_buffer_page(eb, 0);
  3875. __free_extent_buffer(eb);
  3876. }
  3877. static struct extent_buffer *__alloc_extent_buffer(struct extent_io_tree *tree,
  3878. u64 start,
  3879. unsigned long len,
  3880. gfp_t mask)
  3881. {
  3882. struct extent_buffer *eb = NULL;
  3883. eb = kmem_cache_zalloc(extent_buffer_cache, mask);
  3884. if (eb == NULL)
  3885. return NULL;
  3886. eb->start = start;
  3887. eb->len = len;
  3888. eb->tree = tree;
  3889. eb->bflags = 0;
  3890. rwlock_init(&eb->lock);
  3891. atomic_set(&eb->write_locks, 0);
  3892. atomic_set(&eb->read_locks, 0);
  3893. atomic_set(&eb->blocking_readers, 0);
  3894. atomic_set(&eb->blocking_writers, 0);
  3895. atomic_set(&eb->spinning_readers, 0);
  3896. atomic_set(&eb->spinning_writers, 0);
  3897. eb->lock_nested = 0;
  3898. init_waitqueue_head(&eb->write_lock_wq);
  3899. init_waitqueue_head(&eb->read_lock_wq);
  3900. btrfs_leak_debug_add(&eb->leak_list, &buffers);
  3901. spin_lock_init(&eb->refs_lock);
  3902. atomic_set(&eb->refs, 1);
  3903. atomic_set(&eb->io_pages, 0);
  3904. /*
  3905. * Sanity checks, currently the maximum is 64k covered by 16x 4k pages
  3906. */
  3907. BUILD_BUG_ON(BTRFS_MAX_METADATA_BLOCKSIZE
  3908. > MAX_INLINE_EXTENT_BUFFER_SIZE);
  3909. BUG_ON(len > MAX_INLINE_EXTENT_BUFFER_SIZE);
  3910. return eb;
  3911. }
  3912. struct extent_buffer *btrfs_clone_extent_buffer(struct extent_buffer *src)
  3913. {
  3914. unsigned long i;
  3915. struct page *p;
  3916. struct extent_buffer *new;
  3917. unsigned long num_pages = num_extent_pages(src->start, src->len);
  3918. new = __alloc_extent_buffer(NULL, src->start, src->len, GFP_NOFS);
  3919. if (new == NULL)
  3920. return NULL;
  3921. for (i = 0; i < num_pages; i++) {
  3922. p = alloc_page(GFP_NOFS);
  3923. if (!p) {
  3924. btrfs_release_extent_buffer(new);
  3925. return NULL;
  3926. }
  3927. attach_extent_buffer_page(new, p);
  3928. WARN_ON(PageDirty(p));
  3929. SetPageUptodate(p);
  3930. new->pages[i] = p;
  3931. }
  3932. copy_extent_buffer(new, src, 0, 0, src->len);
  3933. set_bit(EXTENT_BUFFER_UPTODATE, &new->bflags);
  3934. set_bit(EXTENT_BUFFER_DUMMY, &new->bflags);
  3935. return new;
  3936. }
  3937. struct extent_buffer *alloc_dummy_extent_buffer(u64 start, unsigned long len)
  3938. {
  3939. struct extent_buffer *eb;
  3940. unsigned long num_pages = num_extent_pages(0, len);
  3941. unsigned long i;
  3942. eb = __alloc_extent_buffer(NULL, start, len, GFP_NOFS);
  3943. if (!eb)
  3944. return NULL;
  3945. for (i = 0; i < num_pages; i++) {
  3946. eb->pages[i] = alloc_page(GFP_NOFS);
  3947. if (!eb->pages[i])
  3948. goto err;
  3949. }
  3950. set_extent_buffer_uptodate(eb);
  3951. btrfs_set_header_nritems(eb, 0);
  3952. set_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
  3953. return eb;
  3954. err:
  3955. for (; i > 0; i--)
  3956. __free_page(eb->pages[i - 1]);
  3957. __free_extent_buffer(eb);
  3958. return NULL;
  3959. }
  3960. static void check_buffer_tree_ref(struct extent_buffer *eb)
  3961. {
  3962. int refs;
  3963. /* the ref bit is tricky. We have to make sure it is set
  3964. * if we have the buffer dirty. Otherwise the
  3965. * code to free a buffer can end up dropping a dirty
  3966. * page
  3967. *
  3968. * Once the ref bit is set, it won't go away while the
  3969. * buffer is dirty or in writeback, and it also won't
  3970. * go away while we have the reference count on the
  3971. * eb bumped.
  3972. *
  3973. * We can't just set the ref bit without bumping the
  3974. * ref on the eb because free_extent_buffer might
  3975. * see the ref bit and try to clear it. If this happens
  3976. * free_extent_buffer might end up dropping our original
  3977. * ref by mistake and freeing the page before we are able
  3978. * to add one more ref.
  3979. *
  3980. * So bump the ref count first, then set the bit. If someone
  3981. * beat us to it, drop the ref we added.
  3982. */
  3983. refs = atomic_read(&eb->refs);
  3984. if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
  3985. return;
  3986. spin_lock(&eb->refs_lock);
  3987. if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
  3988. atomic_inc(&eb->refs);
  3989. spin_unlock(&eb->refs_lock);
  3990. }
  3991. static void mark_extent_buffer_accessed(struct extent_buffer *eb)
  3992. {
  3993. unsigned long num_pages, i;
  3994. check_buffer_tree_ref(eb);
  3995. num_pages = num_extent_pages(eb->start, eb->len);
  3996. for (i = 0; i < num_pages; i++) {
  3997. struct page *p = extent_buffer_page(eb, i);
  3998. mark_page_accessed(p);
  3999. }
  4000. }
  4001. struct extent_buffer *find_extent_buffer(struct extent_io_tree *tree,
  4002. u64 start)
  4003. {
  4004. struct extent_buffer *eb;
  4005. rcu_read_lock();
  4006. eb = radix_tree_lookup(&tree->buffer, start >> PAGE_CACHE_SHIFT);
  4007. if (eb && atomic_inc_not_zero(&eb->refs)) {
  4008. rcu_read_unlock();
  4009. mark_extent_buffer_accessed(eb);
  4010. return eb;
  4011. }
  4012. rcu_read_unlock();
  4013. return NULL;
  4014. }
  4015. struct extent_buffer *alloc_extent_buffer(struct extent_io_tree *tree,
  4016. u64 start, unsigned long len)
  4017. {
  4018. unsigned long num_pages = num_extent_pages(start, len);
  4019. unsigned long i;
  4020. unsigned long index = start >> PAGE_CACHE_SHIFT;
  4021. struct extent_buffer *eb;
  4022. struct extent_buffer *exists = NULL;
  4023. struct page *p;
  4024. struct address_space *mapping = tree->mapping;
  4025. int uptodate = 1;
  4026. int ret;
  4027. eb = find_extent_buffer(tree, start);
  4028. if (eb)
  4029. return eb;
  4030. eb = __alloc_extent_buffer(tree, start, len, GFP_NOFS);
  4031. if (!eb)
  4032. return NULL;
  4033. for (i = 0; i < num_pages; i++, index++) {
  4034. p = find_or_create_page(mapping, index, GFP_NOFS);
  4035. if (!p)
  4036. goto free_eb;
  4037. spin_lock(&mapping->private_lock);
  4038. if (PagePrivate(p)) {
  4039. /*
  4040. * We could have already allocated an eb for this page
  4041. * and attached one so lets see if we can get a ref on
  4042. * the existing eb, and if we can we know it's good and
  4043. * we can just return that one, else we know we can just
  4044. * overwrite page->private.
  4045. */
  4046. exists = (struct extent_buffer *)p->private;
  4047. if (atomic_inc_not_zero(&exists->refs)) {
  4048. spin_unlock(&mapping->private_lock);
  4049. unlock_page(p);
  4050. page_cache_release(p);
  4051. mark_extent_buffer_accessed(exists);
  4052. goto free_eb;
  4053. }
  4054. /*
  4055. * Do this so attach doesn't complain and we need to
  4056. * drop the ref the old guy had.
  4057. */
  4058. ClearPagePrivate(p);
  4059. WARN_ON(PageDirty(p));
  4060. page_cache_release(p);
  4061. }
  4062. attach_extent_buffer_page(eb, p);
  4063. spin_unlock(&mapping->private_lock);
  4064. WARN_ON(PageDirty(p));
  4065. mark_page_accessed(p);
  4066. eb->pages[i] = p;
  4067. if (!PageUptodate(p))
  4068. uptodate = 0;
  4069. /*
  4070. * see below about how we avoid a nasty race with release page
  4071. * and why we unlock later
  4072. */
  4073. }
  4074. if (uptodate)
  4075. set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
  4076. again:
  4077. ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM);
  4078. if (ret)
  4079. goto free_eb;
  4080. spin_lock(&tree->buffer_lock);
  4081. ret = radix_tree_insert(&tree->buffer, start >> PAGE_CACHE_SHIFT, eb);
  4082. spin_unlock(&tree->buffer_lock);
  4083. radix_tree_preload_end();
  4084. if (ret == -EEXIST) {
  4085. exists = find_extent_buffer(tree, start);
  4086. if (exists)
  4087. goto free_eb;
  4088. else
  4089. goto again;
  4090. }
  4091. /* add one reference for the tree */
  4092. check_buffer_tree_ref(eb);
  4093. /*
  4094. * there is a race where release page may have
  4095. * tried to find this extent buffer in the radix
  4096. * but failed. It will tell the VM it is safe to
  4097. * reclaim the, and it will clear the page private bit.
  4098. * We must make sure to set the page private bit properly
  4099. * after the extent buffer is in the radix tree so
  4100. * it doesn't get lost
  4101. */
  4102. SetPageChecked(eb->pages[0]);
  4103. for (i = 1; i < num_pages; i++) {
  4104. p = extent_buffer_page(eb, i);
  4105. ClearPageChecked(p);
  4106. unlock_page(p);
  4107. }
  4108. unlock_page(eb->pages[0]);
  4109. return eb;
  4110. free_eb:
  4111. for (i = 0; i < num_pages; i++) {
  4112. if (eb->pages[i])
  4113. unlock_page(eb->pages[i]);
  4114. }
  4115. WARN_ON(!atomic_dec_and_test(&eb->refs));
  4116. btrfs_release_extent_buffer(eb);
  4117. return exists;
  4118. }
  4119. static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
  4120. {
  4121. struct extent_buffer *eb =
  4122. container_of(head, struct extent_buffer, rcu_head);
  4123. __free_extent_buffer(eb);
  4124. }
  4125. /* Expects to have eb->eb_lock already held */
  4126. static int release_extent_buffer(struct extent_buffer *eb)
  4127. {
  4128. WARN_ON(atomic_read(&eb->refs) == 0);
  4129. if (atomic_dec_and_test(&eb->refs)) {
  4130. if (test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags)) {
  4131. spin_unlock(&eb->refs_lock);
  4132. } else {
  4133. struct extent_io_tree *tree = eb->tree;
  4134. spin_unlock(&eb->refs_lock);
  4135. spin_lock(&tree->buffer_lock);
  4136. radix_tree_delete(&tree->buffer,
  4137. eb->start >> PAGE_CACHE_SHIFT);
  4138. spin_unlock(&tree->buffer_lock);
  4139. }
  4140. /* Should be safe to release our pages at this point */
  4141. btrfs_release_extent_buffer_page(eb, 0);
  4142. call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
  4143. return 1;
  4144. }
  4145. spin_unlock(&eb->refs_lock);
  4146. return 0;
  4147. }
  4148. void free_extent_buffer(struct extent_buffer *eb)
  4149. {
  4150. int refs;
  4151. int old;
  4152. if (!eb)
  4153. return;
  4154. while (1) {
  4155. refs = atomic_read(&eb->refs);
  4156. if (refs <= 3)
  4157. break;
  4158. old = atomic_cmpxchg(&eb->refs, refs, refs - 1);
  4159. if (old == refs)
  4160. return;
  4161. }
  4162. spin_lock(&eb->refs_lock);
  4163. if (atomic_read(&eb->refs) == 2 &&
  4164. test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags))
  4165. atomic_dec(&eb->refs);
  4166. if (atomic_read(&eb->refs) == 2 &&
  4167. test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
  4168. !extent_buffer_under_io(eb) &&
  4169. test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
  4170. atomic_dec(&eb->refs);
  4171. /*
  4172. * I know this is terrible, but it's temporary until we stop tracking
  4173. * the uptodate bits and such for the extent buffers.
  4174. */
  4175. release_extent_buffer(eb);
  4176. }
  4177. void free_extent_buffer_stale(struct extent_buffer *eb)
  4178. {
  4179. if (!eb)
  4180. return;
  4181. spin_lock(&eb->refs_lock);
  4182. set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
  4183. if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
  4184. test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
  4185. atomic_dec(&eb->refs);
  4186. release_extent_buffer(eb);
  4187. }
  4188. void clear_extent_buffer_dirty(struct extent_buffer *eb)
  4189. {
  4190. unsigned long i;
  4191. unsigned long num_pages;
  4192. struct page *page;
  4193. num_pages = num_extent_pages(eb->start, eb->len);
  4194. for (i = 0; i < num_pages; i++) {
  4195. page = extent_buffer_page(eb, i);
  4196. if (!PageDirty(page))
  4197. continue;
  4198. lock_page(page);
  4199. WARN_ON(!PagePrivate(page));
  4200. clear_page_dirty_for_io(page);
  4201. spin_lock_irq(&page->mapping->tree_lock);
  4202. if (!PageDirty(page)) {
  4203. radix_tree_tag_clear(&page->mapping->page_tree,
  4204. page_index(page),
  4205. PAGECACHE_TAG_DIRTY);
  4206. }
  4207. spin_unlock_irq(&page->mapping->tree_lock);
  4208. ClearPageError(page);
  4209. unlock_page(page);
  4210. }
  4211. WARN_ON(atomic_read(&eb->refs) == 0);
  4212. }
  4213. int set_extent_buffer_dirty(struct extent_buffer *eb)
  4214. {
  4215. unsigned long i;
  4216. unsigned long num_pages;
  4217. int was_dirty = 0;
  4218. check_buffer_tree_ref(eb);
  4219. was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
  4220. num_pages = num_extent_pages(eb->start, eb->len);
  4221. WARN_ON(atomic_read(&eb->refs) == 0);
  4222. WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
  4223. for (i = 0; i < num_pages; i++)
  4224. set_page_dirty(extent_buffer_page(eb, i));
  4225. return was_dirty;
  4226. }
  4227. int clear_extent_buffer_uptodate(struct extent_buffer *eb)
  4228. {
  4229. unsigned long i;
  4230. struct page *page;
  4231. unsigned long num_pages;
  4232. clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
  4233. num_pages = num_extent_pages(eb->start, eb->len);
  4234. for (i = 0; i < num_pages; i++) {
  4235. page = extent_buffer_page(eb, i);
  4236. if (page)
  4237. ClearPageUptodate(page);
  4238. }
  4239. return 0;
  4240. }
  4241. int set_extent_buffer_uptodate(struct extent_buffer *eb)
  4242. {
  4243. unsigned long i;
  4244. struct page *page;
  4245. unsigned long num_pages;
  4246. set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
  4247. num_pages = num_extent_pages(eb->start, eb->len);
  4248. for (i = 0; i < num_pages; i++) {
  4249. page = extent_buffer_page(eb, i);
  4250. SetPageUptodate(page);
  4251. }
  4252. return 0;
  4253. }
  4254. int extent_buffer_uptodate(struct extent_buffer *eb)
  4255. {
  4256. return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
  4257. }
  4258. int read_extent_buffer_pages(struct extent_io_tree *tree,
  4259. struct extent_buffer *eb, u64 start, int wait,
  4260. get_extent_t *get_extent, int mirror_num)
  4261. {
  4262. unsigned long i;
  4263. unsigned long start_i;
  4264. struct page *page;
  4265. int err;
  4266. int ret = 0;
  4267. int locked_pages = 0;
  4268. int all_uptodate = 1;
  4269. unsigned long num_pages;
  4270. unsigned long num_reads = 0;
  4271. struct bio *bio = NULL;
  4272. unsigned long bio_flags = 0;
  4273. if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
  4274. return 0;
  4275. if (start) {
  4276. WARN_ON(start < eb->start);
  4277. start_i = (start >> PAGE_CACHE_SHIFT) -
  4278. (eb->start >> PAGE_CACHE_SHIFT);
  4279. } else {
  4280. start_i = 0;
  4281. }
  4282. num_pages = num_extent_pages(eb->start, eb->len);
  4283. for (i = start_i; i < num_pages; i++) {
  4284. page = extent_buffer_page(eb, i);
  4285. if (wait == WAIT_NONE) {
  4286. if (!trylock_page(page))
  4287. goto unlock_exit;
  4288. } else {
  4289. lock_page(page);
  4290. }
  4291. locked_pages++;
  4292. if (!PageUptodate(page)) {
  4293. num_reads++;
  4294. all_uptodate = 0;
  4295. }
  4296. }
  4297. if (all_uptodate) {
  4298. if (start_i == 0)
  4299. set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
  4300. goto unlock_exit;
  4301. }
  4302. clear_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
  4303. eb->read_mirror = 0;
  4304. atomic_set(&eb->io_pages, num_reads);
  4305. for (i = start_i; i < num_pages; i++) {
  4306. page = extent_buffer_page(eb, i);
  4307. if (!PageUptodate(page)) {
  4308. ClearPageError(page);
  4309. err = __extent_read_full_page(tree, page,
  4310. get_extent, &bio,
  4311. mirror_num, &bio_flags,
  4312. READ | REQ_META);
  4313. if (err)
  4314. ret = err;
  4315. } else {
  4316. unlock_page(page);
  4317. }
  4318. }
  4319. if (bio) {
  4320. err = submit_one_bio(READ | REQ_META, bio, mirror_num,
  4321. bio_flags);
  4322. if (err)
  4323. return err;
  4324. }
  4325. if (ret || wait != WAIT_COMPLETE)
  4326. return ret;
  4327. for (i = start_i; i < num_pages; i++) {
  4328. page = extent_buffer_page(eb, i);
  4329. wait_on_page_locked(page);
  4330. if (!PageUptodate(page))
  4331. ret = -EIO;
  4332. }
  4333. return ret;
  4334. unlock_exit:
  4335. i = start_i;
  4336. while (locked_pages > 0) {
  4337. page = extent_buffer_page(eb, i);
  4338. i++;
  4339. unlock_page(page);
  4340. locked_pages--;
  4341. }
  4342. return ret;
  4343. }
  4344. void read_extent_buffer(struct extent_buffer *eb, void *dstv,
  4345. unsigned long start,
  4346. unsigned long len)
  4347. {
  4348. size_t cur;
  4349. size_t offset;
  4350. struct page *page;
  4351. char *kaddr;
  4352. char *dst = (char *)dstv;
  4353. size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
  4354. unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
  4355. WARN_ON(start > eb->len);
  4356. WARN_ON(start + len > eb->start + eb->len);
  4357. offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
  4358. while (len > 0) {
  4359. page = extent_buffer_page(eb, i);
  4360. cur = min(len, (PAGE_CACHE_SIZE - offset));
  4361. kaddr = page_address(page);
  4362. memcpy(dst, kaddr + offset, cur);
  4363. dst += cur;
  4364. len -= cur;
  4365. offset = 0;
  4366. i++;
  4367. }
  4368. }
  4369. int map_private_extent_buffer(struct extent_buffer *eb, unsigned long start,
  4370. unsigned long min_len, char **map,
  4371. unsigned long *map_start,
  4372. unsigned long *map_len)
  4373. {
  4374. size_t offset = start & (PAGE_CACHE_SIZE - 1);
  4375. char *kaddr;
  4376. struct page *p;
  4377. size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
  4378. unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
  4379. unsigned long end_i = (start_offset + start + min_len - 1) >>
  4380. PAGE_CACHE_SHIFT;
  4381. if (i != end_i)
  4382. return -EINVAL;
  4383. if (i == 0) {
  4384. offset = start_offset;
  4385. *map_start = 0;
  4386. } else {
  4387. offset = 0;
  4388. *map_start = ((u64)i << PAGE_CACHE_SHIFT) - start_offset;
  4389. }
  4390. if (start + min_len > eb->len) {
  4391. WARN(1, KERN_ERR "btrfs bad mapping eb start %llu len %lu, "
  4392. "wanted %lu %lu\n",
  4393. eb->start, eb->len, start, min_len);
  4394. return -EINVAL;
  4395. }
  4396. p = extent_buffer_page(eb, i);
  4397. kaddr = page_address(p);
  4398. *map = kaddr + offset;
  4399. *map_len = PAGE_CACHE_SIZE - offset;
  4400. return 0;
  4401. }
  4402. int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv,
  4403. unsigned long start,
  4404. unsigned long len)
  4405. {
  4406. size_t cur;
  4407. size_t offset;
  4408. struct page *page;
  4409. char *kaddr;
  4410. char *ptr = (char *)ptrv;
  4411. size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
  4412. unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
  4413. int ret = 0;
  4414. WARN_ON(start > eb->len);
  4415. WARN_ON(start + len > eb->start + eb->len);
  4416. offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
  4417. while (len > 0) {
  4418. page = extent_buffer_page(eb, i);
  4419. cur = min(len, (PAGE_CACHE_SIZE - offset));
  4420. kaddr = page_address(page);
  4421. ret = memcmp(ptr, kaddr + offset, cur);
  4422. if (ret)
  4423. break;
  4424. ptr += cur;
  4425. len -= cur;
  4426. offset = 0;
  4427. i++;
  4428. }
  4429. return ret;
  4430. }
  4431. void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
  4432. unsigned long start, unsigned long len)
  4433. {
  4434. size_t cur;
  4435. size_t offset;
  4436. struct page *page;
  4437. char *kaddr;
  4438. char *src = (char *)srcv;
  4439. size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
  4440. unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
  4441. WARN_ON(start > eb->len);
  4442. WARN_ON(start + len > eb->start + eb->len);
  4443. offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
  4444. while (len > 0) {
  4445. page = extent_buffer_page(eb, i);
  4446. WARN_ON(!PageUptodate(page));
  4447. cur = min(len, PAGE_CACHE_SIZE - offset);
  4448. kaddr = page_address(page);
  4449. memcpy(kaddr + offset, src, cur);
  4450. src += cur;
  4451. len -= cur;
  4452. offset = 0;
  4453. i++;
  4454. }
  4455. }
  4456. void memset_extent_buffer(struct extent_buffer *eb, char c,
  4457. unsigned long start, unsigned long len)
  4458. {
  4459. size_t cur;
  4460. size_t offset;
  4461. struct page *page;
  4462. char *kaddr;
  4463. size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
  4464. unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
  4465. WARN_ON(start > eb->len);
  4466. WARN_ON(start + len > eb->start + eb->len);
  4467. offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
  4468. while (len > 0) {
  4469. page = extent_buffer_page(eb, i);
  4470. WARN_ON(!PageUptodate(page));
  4471. cur = min(len, PAGE_CACHE_SIZE - offset);
  4472. kaddr = page_address(page);
  4473. memset(kaddr + offset, c, cur);
  4474. len -= cur;
  4475. offset = 0;
  4476. i++;
  4477. }
  4478. }
  4479. void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
  4480. unsigned long dst_offset, unsigned long src_offset,
  4481. unsigned long len)
  4482. {
  4483. u64 dst_len = dst->len;
  4484. size_t cur;
  4485. size_t offset;
  4486. struct page *page;
  4487. char *kaddr;
  4488. size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
  4489. unsigned long i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
  4490. WARN_ON(src->len != dst_len);
  4491. offset = (start_offset + dst_offset) &
  4492. (PAGE_CACHE_SIZE - 1);
  4493. while (len > 0) {
  4494. page = extent_buffer_page(dst, i);
  4495. WARN_ON(!PageUptodate(page));
  4496. cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset));
  4497. kaddr = page_address(page);
  4498. read_extent_buffer(src, kaddr + offset, src_offset, cur);
  4499. src_offset += cur;
  4500. len -= cur;
  4501. offset = 0;
  4502. i++;
  4503. }
  4504. }
  4505. static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
  4506. {
  4507. unsigned long distance = (src > dst) ? src - dst : dst - src;
  4508. return distance < len;
  4509. }
  4510. static void copy_pages(struct page *dst_page, struct page *src_page,
  4511. unsigned long dst_off, unsigned long src_off,
  4512. unsigned long len)
  4513. {
  4514. char *dst_kaddr = page_address(dst_page);
  4515. char *src_kaddr;
  4516. int must_memmove = 0;
  4517. if (dst_page != src_page) {
  4518. src_kaddr = page_address(src_page);
  4519. } else {
  4520. src_kaddr = dst_kaddr;
  4521. if (areas_overlap(src_off, dst_off, len))
  4522. must_memmove = 1;
  4523. }
  4524. if (must_memmove)
  4525. memmove(dst_kaddr + dst_off, src_kaddr + src_off, len);
  4526. else
  4527. memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
  4528. }
  4529. void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
  4530. unsigned long src_offset, unsigned long len)
  4531. {
  4532. size_t cur;
  4533. size_t dst_off_in_page;
  4534. size_t src_off_in_page;
  4535. size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
  4536. unsigned long dst_i;
  4537. unsigned long src_i;
  4538. if (src_offset + len > dst->len) {
  4539. printk(KERN_ERR "btrfs memmove bogus src_offset %lu move "
  4540. "len %lu dst len %lu\n", src_offset, len, dst->len);
  4541. BUG_ON(1);
  4542. }
  4543. if (dst_offset + len > dst->len) {
  4544. printk(KERN_ERR "btrfs memmove bogus dst_offset %lu move "
  4545. "len %lu dst len %lu\n", dst_offset, len, dst->len);
  4546. BUG_ON(1);
  4547. }
  4548. while (len > 0) {
  4549. dst_off_in_page = (start_offset + dst_offset) &
  4550. (PAGE_CACHE_SIZE - 1);
  4551. src_off_in_page = (start_offset + src_offset) &
  4552. (PAGE_CACHE_SIZE - 1);
  4553. dst_i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
  4554. src_i = (start_offset + src_offset) >> PAGE_CACHE_SHIFT;
  4555. cur = min(len, (unsigned long)(PAGE_CACHE_SIZE -
  4556. src_off_in_page));
  4557. cur = min_t(unsigned long, cur,
  4558. (unsigned long)(PAGE_CACHE_SIZE - dst_off_in_page));
  4559. copy_pages(extent_buffer_page(dst, dst_i),
  4560. extent_buffer_page(dst, src_i),
  4561. dst_off_in_page, src_off_in_page, cur);
  4562. src_offset += cur;
  4563. dst_offset += cur;
  4564. len -= cur;
  4565. }
  4566. }
  4567. void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
  4568. unsigned long src_offset, unsigned long len)
  4569. {
  4570. size_t cur;
  4571. size_t dst_off_in_page;
  4572. size_t src_off_in_page;
  4573. unsigned long dst_end = dst_offset + len - 1;
  4574. unsigned long src_end = src_offset + len - 1;
  4575. size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
  4576. unsigned long dst_i;
  4577. unsigned long src_i;
  4578. if (src_offset + len > dst->len) {
  4579. printk(KERN_ERR "btrfs memmove bogus src_offset %lu move "
  4580. "len %lu len %lu\n", src_offset, len, dst->len);
  4581. BUG_ON(1);
  4582. }
  4583. if (dst_offset + len > dst->len) {
  4584. printk(KERN_ERR "btrfs memmove bogus dst_offset %lu move "
  4585. "len %lu len %lu\n", dst_offset, len, dst->len);
  4586. BUG_ON(1);
  4587. }
  4588. if (dst_offset < src_offset) {
  4589. memcpy_extent_buffer(dst, dst_offset, src_offset, len);
  4590. return;
  4591. }
  4592. while (len > 0) {
  4593. dst_i = (start_offset + dst_end) >> PAGE_CACHE_SHIFT;
  4594. src_i = (start_offset + src_end) >> PAGE_CACHE_SHIFT;
  4595. dst_off_in_page = (start_offset + dst_end) &
  4596. (PAGE_CACHE_SIZE - 1);
  4597. src_off_in_page = (start_offset + src_end) &
  4598. (PAGE_CACHE_SIZE - 1);
  4599. cur = min_t(unsigned long, len, src_off_in_page + 1);
  4600. cur = min(cur, dst_off_in_page + 1);
  4601. copy_pages(extent_buffer_page(dst, dst_i),
  4602. extent_buffer_page(dst, src_i),
  4603. dst_off_in_page - cur + 1,
  4604. src_off_in_page - cur + 1, cur);
  4605. dst_end -= cur;
  4606. src_end -= cur;
  4607. len -= cur;
  4608. }
  4609. }
  4610. int try_release_extent_buffer(struct page *page)
  4611. {
  4612. struct extent_buffer *eb;
  4613. /*
  4614. * We need to make sure noboody is attaching this page to an eb right
  4615. * now.
  4616. */
  4617. spin_lock(&page->mapping->private_lock);
  4618. if (!PagePrivate(page)) {
  4619. spin_unlock(&page->mapping->private_lock);
  4620. return 1;
  4621. }
  4622. eb = (struct extent_buffer *)page->private;
  4623. BUG_ON(!eb);
  4624. /*
  4625. * This is a little awful but should be ok, we need to make sure that
  4626. * the eb doesn't disappear out from under us while we're looking at
  4627. * this page.
  4628. */
  4629. spin_lock(&eb->refs_lock);
  4630. if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
  4631. spin_unlock(&eb->refs_lock);
  4632. spin_unlock(&page->mapping->private_lock);
  4633. return 0;
  4634. }
  4635. spin_unlock(&page->mapping->private_lock);
  4636. /*
  4637. * If tree ref isn't set then we know the ref on this eb is a real ref,
  4638. * so just return, this page will likely be freed soon anyway.
  4639. */
  4640. if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
  4641. spin_unlock(&eb->refs_lock);
  4642. return 0;
  4643. }
  4644. return release_extent_buffer(eb);
  4645. }