extent-tree.c 233 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788778977907791779277937794779577967797779877997800780178027803780478057806780778087809781078117812781378147815781678177818781978207821782278237824782578267827782878297830783178327833783478357836783778387839784078417842784378447845784678477848784978507851785278537854785578567857785878597860786178627863786478657866786778687869787078717872787378747875787678777878787978807881788278837884788578867887788878897890789178927893789478957896789778987899790079017902790379047905790679077908790979107911791279137914791579167917791879197920792179227923792479257926792779287929793079317932793379347935793679377938793979407941794279437944794579467947794879497950795179527953795479557956795779587959796079617962796379647965796679677968796979707971797279737974797579767977797879797980798179827983798479857986798779887989799079917992799379947995799679977998799980008001800280038004800580068007800880098010801180128013801480158016801780188019802080218022802380248025802680278028802980308031803280338034803580368037803880398040804180428043804480458046804780488049805080518052805380548055805680578058805980608061806280638064806580668067806880698070807180728073807480758076807780788079808080818082808380848085808680878088808980908091809280938094809580968097809880998100810181028103810481058106810781088109811081118112811381148115811681178118811981208121812281238124812581268127812881298130813181328133813481358136813781388139814081418142814381448145814681478148814981508151815281538154815581568157815881598160816181628163816481658166816781688169817081718172817381748175817681778178817981808181818281838184818581868187818881898190819181928193819481958196819781988199820082018202820382048205820682078208820982108211821282138214821582168217821882198220822182228223822482258226822782288229823082318232823382348235823682378238823982408241824282438244824582468247824882498250825182528253825482558256825782588259826082618262826382648265826682678268826982708271827282738274827582768277827882798280828182828283828482858286828782888289829082918292829382948295829682978298829983008301830283038304830583068307830883098310831183128313831483158316831783188319832083218322832383248325832683278328832983308331833283338334833583368337833883398340834183428343834483458346834783488349835083518352835383548355835683578358835983608361836283638364836583668367836883698370837183728373837483758376837783788379838083818382838383848385838683878388838983908391839283938394839583968397839883998400840184028403840484058406840784088409841084118412841384148415841684178418841984208421842284238424842584268427842884298430843184328433843484358436843784388439844084418442844384448445844684478448844984508451845284538454845584568457845884598460846184628463846484658466846784688469847084718472847384748475847684778478847984808481848284838484848584868487848884898490849184928493849484958496849784988499850085018502850385048505850685078508850985108511851285138514851585168517851885198520852185228523852485258526852785288529853085318532853385348535853685378538853985408541854285438544854585468547854885498550855185528553855485558556855785588559856085618562856385648565856685678568856985708571857285738574857585768577857885798580858185828583858485858586858785888589859085918592859385948595859685978598859986008601860286038604860586068607860886098610861186128613861486158616861786188619862086218622862386248625862686278628862986308631863286338634863586368637863886398640864186428643864486458646864786488649865086518652865386548655865686578658865986608661866286638664866586668667866886698670867186728673867486758676867786788679868086818682868386848685868686878688868986908691869286938694869586968697869886998700870187028703870487058706870787088709871087118712871387148715871687178718871987208721872287238724872587268727872887298730873187328733873487358736873787388739874087418742874387448745874687478748874987508751875287538754875587568757875887598760876187628763876487658766876787688769877087718772877387748775877687778778877987808781878287838784878587868787878887898790879187928793879487958796879787988799880088018802880388048805880688078808880988108811881288138814881588168817881888198820882188228823882488258826882788288829883088318832883388348835883688378838883988408841884288438844884588468847884888498850885188528853885488558856885788588859886088618862886388648865886688678868886988708871887288738874887588768877887888798880888188828883888488858886888788888889889088918892889388948895889688978898889989008901890289038904890589068907890889098910891189128913891489158916891789188919892089218922892389248925892689278928892989308931893289338934
  1. /*
  2. * Copyright (C) 2007 Oracle. All rights reserved.
  3. *
  4. * This program is free software; you can redistribute it and/or
  5. * modify it under the terms of the GNU General Public
  6. * License v2 as published by the Free Software Foundation.
  7. *
  8. * This program is distributed in the hope that it will be useful,
  9. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11. * General Public License for more details.
  12. *
  13. * You should have received a copy of the GNU General Public
  14. * License along with this program; if not, write to the
  15. * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  16. * Boston, MA 021110-1307, USA.
  17. */
  18. #include <linux/sched.h>
  19. #include <linux/pagemap.h>
  20. #include <linux/writeback.h>
  21. #include <linux/blkdev.h>
  22. #include <linux/sort.h>
  23. #include <linux/rcupdate.h>
  24. #include <linux/kthread.h>
  25. #include <linux/slab.h>
  26. #include "compat.h"
  27. #include "hash.h"
  28. #include "ctree.h"
  29. #include "disk-io.h"
  30. #include "print-tree.h"
  31. #include "transaction.h"
  32. #include "volumes.h"
  33. #include "locking.h"
  34. #include "free-space-cache.h"
  35. /* control flags for do_chunk_alloc's force field
  36. * CHUNK_ALLOC_NO_FORCE means to only allocate a chunk
  37. * if we really need one.
  38. *
  39. * CHUNK_ALLOC_FORCE means it must try to allocate one
  40. *
  41. * CHUNK_ALLOC_LIMITED means to only try and allocate one
  42. * if we have very few chunks already allocated. This is
  43. * used as part of the clustering code to help make sure
  44. * we have a good pool of storage to cluster in, without
  45. * filling the FS with empty chunks
  46. *
  47. */
  48. enum {
  49. CHUNK_ALLOC_NO_FORCE = 0,
  50. CHUNK_ALLOC_FORCE = 1,
  51. CHUNK_ALLOC_LIMITED = 2,
  52. };
  53. static int update_block_group(struct btrfs_trans_handle *trans,
  54. struct btrfs_root *root,
  55. u64 bytenr, u64 num_bytes, int alloc);
  56. static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
  57. struct btrfs_root *root,
  58. u64 bytenr, u64 num_bytes, u64 parent,
  59. u64 root_objectid, u64 owner_objectid,
  60. u64 owner_offset, int refs_to_drop,
  61. struct btrfs_delayed_extent_op *extra_op);
  62. static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
  63. struct extent_buffer *leaf,
  64. struct btrfs_extent_item *ei);
  65. static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
  66. struct btrfs_root *root,
  67. u64 parent, u64 root_objectid,
  68. u64 flags, u64 owner, u64 offset,
  69. struct btrfs_key *ins, int ref_mod);
  70. static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
  71. struct btrfs_root *root,
  72. u64 parent, u64 root_objectid,
  73. u64 flags, struct btrfs_disk_key *key,
  74. int level, struct btrfs_key *ins);
  75. static int do_chunk_alloc(struct btrfs_trans_handle *trans,
  76. struct btrfs_root *extent_root, u64 alloc_bytes,
  77. u64 flags, int force);
  78. static int find_next_key(struct btrfs_path *path, int level,
  79. struct btrfs_key *key);
  80. static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
  81. int dump_block_groups);
  82. static noinline int
  83. block_group_cache_done(struct btrfs_block_group_cache *cache)
  84. {
  85. smp_mb();
  86. return cache->cached == BTRFS_CACHE_FINISHED;
  87. }
  88. static int block_group_bits(struct btrfs_block_group_cache *cache, u64 bits)
  89. {
  90. return (cache->flags & bits) == bits;
  91. }
  92. void btrfs_get_block_group(struct btrfs_block_group_cache *cache)
  93. {
  94. atomic_inc(&cache->count);
  95. }
  96. void btrfs_put_block_group(struct btrfs_block_group_cache *cache)
  97. {
  98. if (atomic_dec_and_test(&cache->count)) {
  99. WARN_ON(cache->pinned > 0);
  100. WARN_ON(cache->reserved > 0);
  101. WARN_ON(cache->reserved_pinned > 0);
  102. kfree(cache->free_space_ctl);
  103. kfree(cache);
  104. }
  105. }
  106. /*
  107. * this adds the block group to the fs_info rb tree for the block group
  108. * cache
  109. */
  110. static int btrfs_add_block_group_cache(struct btrfs_fs_info *info,
  111. struct btrfs_block_group_cache *block_group)
  112. {
  113. struct rb_node **p;
  114. struct rb_node *parent = NULL;
  115. struct btrfs_block_group_cache *cache;
  116. spin_lock(&info->block_group_cache_lock);
  117. p = &info->block_group_cache_tree.rb_node;
  118. while (*p) {
  119. parent = *p;
  120. cache = rb_entry(parent, struct btrfs_block_group_cache,
  121. cache_node);
  122. if (block_group->key.objectid < cache->key.objectid) {
  123. p = &(*p)->rb_left;
  124. } else if (block_group->key.objectid > cache->key.objectid) {
  125. p = &(*p)->rb_right;
  126. } else {
  127. spin_unlock(&info->block_group_cache_lock);
  128. return -EEXIST;
  129. }
  130. }
  131. rb_link_node(&block_group->cache_node, parent, p);
  132. rb_insert_color(&block_group->cache_node,
  133. &info->block_group_cache_tree);
  134. spin_unlock(&info->block_group_cache_lock);
  135. return 0;
  136. }
  137. /*
  138. * This will return the block group at or after bytenr if contains is 0, else
  139. * it will return the block group that contains the bytenr
  140. */
  141. static struct btrfs_block_group_cache *
  142. block_group_cache_tree_search(struct btrfs_fs_info *info, u64 bytenr,
  143. int contains)
  144. {
  145. struct btrfs_block_group_cache *cache, *ret = NULL;
  146. struct rb_node *n;
  147. u64 end, start;
  148. spin_lock(&info->block_group_cache_lock);
  149. n = info->block_group_cache_tree.rb_node;
  150. while (n) {
  151. cache = rb_entry(n, struct btrfs_block_group_cache,
  152. cache_node);
  153. end = cache->key.objectid + cache->key.offset - 1;
  154. start = cache->key.objectid;
  155. if (bytenr < start) {
  156. if (!contains && (!ret || start < ret->key.objectid))
  157. ret = cache;
  158. n = n->rb_left;
  159. } else if (bytenr > start) {
  160. if (contains && bytenr <= end) {
  161. ret = cache;
  162. break;
  163. }
  164. n = n->rb_right;
  165. } else {
  166. ret = cache;
  167. break;
  168. }
  169. }
  170. if (ret)
  171. btrfs_get_block_group(ret);
  172. spin_unlock(&info->block_group_cache_lock);
  173. return ret;
  174. }
  175. static int add_excluded_extent(struct btrfs_root *root,
  176. u64 start, u64 num_bytes)
  177. {
  178. u64 end = start + num_bytes - 1;
  179. set_extent_bits(&root->fs_info->freed_extents[0],
  180. start, end, EXTENT_UPTODATE, GFP_NOFS);
  181. set_extent_bits(&root->fs_info->freed_extents[1],
  182. start, end, EXTENT_UPTODATE, GFP_NOFS);
  183. return 0;
  184. }
  185. static void free_excluded_extents(struct btrfs_root *root,
  186. struct btrfs_block_group_cache *cache)
  187. {
  188. u64 start, end;
  189. start = cache->key.objectid;
  190. end = start + cache->key.offset - 1;
  191. clear_extent_bits(&root->fs_info->freed_extents[0],
  192. start, end, EXTENT_UPTODATE, GFP_NOFS);
  193. clear_extent_bits(&root->fs_info->freed_extents[1],
  194. start, end, EXTENT_UPTODATE, GFP_NOFS);
  195. }
  196. static int exclude_super_stripes(struct btrfs_root *root,
  197. struct btrfs_block_group_cache *cache)
  198. {
  199. u64 bytenr;
  200. u64 *logical;
  201. int stripe_len;
  202. int i, nr, ret;
  203. if (cache->key.objectid < BTRFS_SUPER_INFO_OFFSET) {
  204. stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->key.objectid;
  205. cache->bytes_super += stripe_len;
  206. ret = add_excluded_extent(root, cache->key.objectid,
  207. stripe_len);
  208. BUG_ON(ret);
  209. }
  210. for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
  211. bytenr = btrfs_sb_offset(i);
  212. ret = btrfs_rmap_block(&root->fs_info->mapping_tree,
  213. cache->key.objectid, bytenr,
  214. 0, &logical, &nr, &stripe_len);
  215. BUG_ON(ret);
  216. while (nr--) {
  217. cache->bytes_super += stripe_len;
  218. ret = add_excluded_extent(root, logical[nr],
  219. stripe_len);
  220. BUG_ON(ret);
  221. }
  222. kfree(logical);
  223. }
  224. return 0;
  225. }
  226. static struct btrfs_caching_control *
  227. get_caching_control(struct btrfs_block_group_cache *cache)
  228. {
  229. struct btrfs_caching_control *ctl;
  230. spin_lock(&cache->lock);
  231. if (cache->cached != BTRFS_CACHE_STARTED) {
  232. spin_unlock(&cache->lock);
  233. return NULL;
  234. }
  235. /* We're loading it the fast way, so we don't have a caching_ctl. */
  236. if (!cache->caching_ctl) {
  237. spin_unlock(&cache->lock);
  238. return NULL;
  239. }
  240. ctl = cache->caching_ctl;
  241. atomic_inc(&ctl->count);
  242. spin_unlock(&cache->lock);
  243. return ctl;
  244. }
  245. static void put_caching_control(struct btrfs_caching_control *ctl)
  246. {
  247. if (atomic_dec_and_test(&ctl->count))
  248. kfree(ctl);
  249. }
  250. /*
  251. * this is only called by cache_block_group, since we could have freed extents
  252. * we need to check the pinned_extents for any extents that can't be used yet
  253. * since their free space will be released as soon as the transaction commits.
  254. */
  255. static u64 add_new_free_space(struct btrfs_block_group_cache *block_group,
  256. struct btrfs_fs_info *info, u64 start, u64 end)
  257. {
  258. u64 extent_start, extent_end, size, total_added = 0;
  259. int ret;
  260. while (start < end) {
  261. ret = find_first_extent_bit(info->pinned_extents, start,
  262. &extent_start, &extent_end,
  263. EXTENT_DIRTY | EXTENT_UPTODATE);
  264. if (ret)
  265. break;
  266. if (extent_start <= start) {
  267. start = extent_end + 1;
  268. } else if (extent_start > start && extent_start < end) {
  269. size = extent_start - start;
  270. total_added += size;
  271. ret = btrfs_add_free_space(block_group, start,
  272. size);
  273. BUG_ON(ret);
  274. start = extent_end + 1;
  275. } else {
  276. break;
  277. }
  278. }
  279. if (start < end) {
  280. size = end - start;
  281. total_added += size;
  282. ret = btrfs_add_free_space(block_group, start, size);
  283. BUG_ON(ret);
  284. }
  285. return total_added;
  286. }
  287. static int caching_kthread(void *data)
  288. {
  289. struct btrfs_block_group_cache *block_group = data;
  290. struct btrfs_fs_info *fs_info = block_group->fs_info;
  291. struct btrfs_caching_control *caching_ctl = block_group->caching_ctl;
  292. struct btrfs_root *extent_root = fs_info->extent_root;
  293. struct btrfs_path *path;
  294. struct extent_buffer *leaf;
  295. struct btrfs_key key;
  296. u64 total_found = 0;
  297. u64 last = 0;
  298. u32 nritems;
  299. int ret = 0;
  300. path = btrfs_alloc_path();
  301. if (!path)
  302. return -ENOMEM;
  303. last = max_t(u64, block_group->key.objectid, BTRFS_SUPER_INFO_OFFSET);
  304. /*
  305. * We don't want to deadlock with somebody trying to allocate a new
  306. * extent for the extent root while also trying to search the extent
  307. * root to add free space. So we skip locking and search the commit
  308. * root, since its read-only
  309. */
  310. path->skip_locking = 1;
  311. path->search_commit_root = 1;
  312. path->reada = 2;
  313. key.objectid = last;
  314. key.offset = 0;
  315. key.type = BTRFS_EXTENT_ITEM_KEY;
  316. again:
  317. mutex_lock(&caching_ctl->mutex);
  318. /* need to make sure the commit_root doesn't disappear */
  319. down_read(&fs_info->extent_commit_sem);
  320. ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
  321. if (ret < 0)
  322. goto err;
  323. leaf = path->nodes[0];
  324. nritems = btrfs_header_nritems(leaf);
  325. while (1) {
  326. smp_mb();
  327. if (fs_info->closing > 1) {
  328. last = (u64)-1;
  329. break;
  330. }
  331. if (path->slots[0] < nritems) {
  332. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  333. } else {
  334. ret = find_next_key(path, 0, &key);
  335. if (ret)
  336. break;
  337. caching_ctl->progress = last;
  338. btrfs_release_path(extent_root, path);
  339. up_read(&fs_info->extent_commit_sem);
  340. mutex_unlock(&caching_ctl->mutex);
  341. if (btrfs_transaction_in_commit(fs_info))
  342. schedule_timeout(1);
  343. else
  344. cond_resched();
  345. goto again;
  346. }
  347. if (key.objectid < block_group->key.objectid) {
  348. path->slots[0]++;
  349. continue;
  350. }
  351. if (key.objectid >= block_group->key.objectid +
  352. block_group->key.offset)
  353. break;
  354. if (key.type == BTRFS_EXTENT_ITEM_KEY) {
  355. total_found += add_new_free_space(block_group,
  356. fs_info, last,
  357. key.objectid);
  358. last = key.objectid + key.offset;
  359. if (total_found > (1024 * 1024 * 2)) {
  360. total_found = 0;
  361. wake_up(&caching_ctl->wait);
  362. }
  363. }
  364. path->slots[0]++;
  365. }
  366. ret = 0;
  367. total_found += add_new_free_space(block_group, fs_info, last,
  368. block_group->key.objectid +
  369. block_group->key.offset);
  370. caching_ctl->progress = (u64)-1;
  371. spin_lock(&block_group->lock);
  372. block_group->caching_ctl = NULL;
  373. block_group->cached = BTRFS_CACHE_FINISHED;
  374. spin_unlock(&block_group->lock);
  375. err:
  376. btrfs_free_path(path);
  377. up_read(&fs_info->extent_commit_sem);
  378. free_excluded_extents(extent_root, block_group);
  379. mutex_unlock(&caching_ctl->mutex);
  380. wake_up(&caching_ctl->wait);
  381. put_caching_control(caching_ctl);
  382. atomic_dec(&block_group->space_info->caching_threads);
  383. btrfs_put_block_group(block_group);
  384. return 0;
  385. }
  386. static int cache_block_group(struct btrfs_block_group_cache *cache,
  387. struct btrfs_trans_handle *trans,
  388. struct btrfs_root *root,
  389. int load_cache_only)
  390. {
  391. struct btrfs_fs_info *fs_info = cache->fs_info;
  392. struct btrfs_caching_control *caching_ctl;
  393. struct task_struct *tsk;
  394. int ret = 0;
  395. smp_mb();
  396. if (cache->cached != BTRFS_CACHE_NO)
  397. return 0;
  398. /*
  399. * We can't do the read from on-disk cache during a commit since we need
  400. * to have the normal tree locking. Also if we are currently trying to
  401. * allocate blocks for the tree root we can't do the fast caching since
  402. * we likely hold important locks.
  403. */
  404. if (trans && (!trans->transaction->in_commit) &&
  405. (root && root != root->fs_info->tree_root)) {
  406. spin_lock(&cache->lock);
  407. if (cache->cached != BTRFS_CACHE_NO) {
  408. spin_unlock(&cache->lock);
  409. return 0;
  410. }
  411. cache->cached = BTRFS_CACHE_STARTED;
  412. spin_unlock(&cache->lock);
  413. ret = load_free_space_cache(fs_info, cache);
  414. spin_lock(&cache->lock);
  415. if (ret == 1) {
  416. cache->cached = BTRFS_CACHE_FINISHED;
  417. cache->last_byte_to_unpin = (u64)-1;
  418. } else {
  419. cache->cached = BTRFS_CACHE_NO;
  420. }
  421. spin_unlock(&cache->lock);
  422. if (ret == 1) {
  423. free_excluded_extents(fs_info->extent_root, cache);
  424. return 0;
  425. }
  426. }
  427. if (load_cache_only)
  428. return 0;
  429. caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_NOFS);
  430. BUG_ON(!caching_ctl);
  431. INIT_LIST_HEAD(&caching_ctl->list);
  432. mutex_init(&caching_ctl->mutex);
  433. init_waitqueue_head(&caching_ctl->wait);
  434. caching_ctl->block_group = cache;
  435. caching_ctl->progress = cache->key.objectid;
  436. /* one for caching kthread, one for caching block group list */
  437. atomic_set(&caching_ctl->count, 2);
  438. spin_lock(&cache->lock);
  439. if (cache->cached != BTRFS_CACHE_NO) {
  440. spin_unlock(&cache->lock);
  441. kfree(caching_ctl);
  442. return 0;
  443. }
  444. cache->caching_ctl = caching_ctl;
  445. cache->cached = BTRFS_CACHE_STARTED;
  446. spin_unlock(&cache->lock);
  447. down_write(&fs_info->extent_commit_sem);
  448. list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups);
  449. up_write(&fs_info->extent_commit_sem);
  450. atomic_inc(&cache->space_info->caching_threads);
  451. btrfs_get_block_group(cache);
  452. tsk = kthread_run(caching_kthread, cache, "btrfs-cache-%llu\n",
  453. cache->key.objectid);
  454. if (IS_ERR(tsk)) {
  455. ret = PTR_ERR(tsk);
  456. printk(KERN_ERR "error running thread %d\n", ret);
  457. BUG();
  458. }
  459. return ret;
  460. }
  461. /*
  462. * return the block group that starts at or after bytenr
  463. */
  464. static struct btrfs_block_group_cache *
  465. btrfs_lookup_first_block_group(struct btrfs_fs_info *info, u64 bytenr)
  466. {
  467. struct btrfs_block_group_cache *cache;
  468. cache = block_group_cache_tree_search(info, bytenr, 0);
  469. return cache;
  470. }
  471. /*
  472. * return the block group that contains the given bytenr
  473. */
  474. struct btrfs_block_group_cache *btrfs_lookup_block_group(
  475. struct btrfs_fs_info *info,
  476. u64 bytenr)
  477. {
  478. struct btrfs_block_group_cache *cache;
  479. cache = block_group_cache_tree_search(info, bytenr, 1);
  480. return cache;
  481. }
  482. static struct btrfs_space_info *__find_space_info(struct btrfs_fs_info *info,
  483. u64 flags)
  484. {
  485. struct list_head *head = &info->space_info;
  486. struct btrfs_space_info *found;
  487. flags &= BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_SYSTEM |
  488. BTRFS_BLOCK_GROUP_METADATA;
  489. rcu_read_lock();
  490. list_for_each_entry_rcu(found, head, list) {
  491. if (found->flags & flags) {
  492. rcu_read_unlock();
  493. return found;
  494. }
  495. }
  496. rcu_read_unlock();
  497. return NULL;
  498. }
  499. /*
  500. * after adding space to the filesystem, we need to clear the full flags
  501. * on all the space infos.
  502. */
  503. void btrfs_clear_space_info_full(struct btrfs_fs_info *info)
  504. {
  505. struct list_head *head = &info->space_info;
  506. struct btrfs_space_info *found;
  507. rcu_read_lock();
  508. list_for_each_entry_rcu(found, head, list)
  509. found->full = 0;
  510. rcu_read_unlock();
  511. }
  512. static u64 div_factor(u64 num, int factor)
  513. {
  514. if (factor == 10)
  515. return num;
  516. num *= factor;
  517. do_div(num, 10);
  518. return num;
  519. }
  520. static u64 div_factor_fine(u64 num, int factor)
  521. {
  522. if (factor == 100)
  523. return num;
  524. num *= factor;
  525. do_div(num, 100);
  526. return num;
  527. }
  528. u64 btrfs_find_block_group(struct btrfs_root *root,
  529. u64 search_start, u64 search_hint, int owner)
  530. {
  531. struct btrfs_block_group_cache *cache;
  532. u64 used;
  533. u64 last = max(search_hint, search_start);
  534. u64 group_start = 0;
  535. int full_search = 0;
  536. int factor = 9;
  537. int wrapped = 0;
  538. again:
  539. while (1) {
  540. cache = btrfs_lookup_first_block_group(root->fs_info, last);
  541. if (!cache)
  542. break;
  543. spin_lock(&cache->lock);
  544. last = cache->key.objectid + cache->key.offset;
  545. used = btrfs_block_group_used(&cache->item);
  546. if ((full_search || !cache->ro) &&
  547. block_group_bits(cache, BTRFS_BLOCK_GROUP_METADATA)) {
  548. if (used + cache->pinned + cache->reserved <
  549. div_factor(cache->key.offset, factor)) {
  550. group_start = cache->key.objectid;
  551. spin_unlock(&cache->lock);
  552. btrfs_put_block_group(cache);
  553. goto found;
  554. }
  555. }
  556. spin_unlock(&cache->lock);
  557. btrfs_put_block_group(cache);
  558. cond_resched();
  559. }
  560. if (!wrapped) {
  561. last = search_start;
  562. wrapped = 1;
  563. goto again;
  564. }
  565. if (!full_search && factor < 10) {
  566. last = search_start;
  567. full_search = 1;
  568. factor = 10;
  569. goto again;
  570. }
  571. found:
  572. return group_start;
  573. }
  574. /* simple helper to search for an existing extent at a given offset */
  575. int btrfs_lookup_extent(struct btrfs_root *root, u64 start, u64 len)
  576. {
  577. int ret;
  578. struct btrfs_key key;
  579. struct btrfs_path *path;
  580. path = btrfs_alloc_path();
  581. BUG_ON(!path);
  582. key.objectid = start;
  583. key.offset = len;
  584. btrfs_set_key_type(&key, BTRFS_EXTENT_ITEM_KEY);
  585. ret = btrfs_search_slot(NULL, root->fs_info->extent_root, &key, path,
  586. 0, 0);
  587. btrfs_free_path(path);
  588. return ret;
  589. }
  590. /*
  591. * helper function to lookup reference count and flags of extent.
  592. *
  593. * the head node for delayed ref is used to store the sum of all the
  594. * reference count modifications queued up in the rbtree. the head
  595. * node may also store the extent flags to set. This way you can check
  596. * to see what the reference count and extent flags would be if all of
  597. * the delayed refs are not processed.
  598. */
  599. int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans,
  600. struct btrfs_root *root, u64 bytenr,
  601. u64 num_bytes, u64 *refs, u64 *flags)
  602. {
  603. struct btrfs_delayed_ref_head *head;
  604. struct btrfs_delayed_ref_root *delayed_refs;
  605. struct btrfs_path *path;
  606. struct btrfs_extent_item *ei;
  607. struct extent_buffer *leaf;
  608. struct btrfs_key key;
  609. u32 item_size;
  610. u64 num_refs;
  611. u64 extent_flags;
  612. int ret;
  613. path = btrfs_alloc_path();
  614. if (!path)
  615. return -ENOMEM;
  616. key.objectid = bytenr;
  617. key.type = BTRFS_EXTENT_ITEM_KEY;
  618. key.offset = num_bytes;
  619. if (!trans) {
  620. path->skip_locking = 1;
  621. path->search_commit_root = 1;
  622. }
  623. again:
  624. ret = btrfs_search_slot(trans, root->fs_info->extent_root,
  625. &key, path, 0, 0);
  626. if (ret < 0)
  627. goto out_free;
  628. if (ret == 0) {
  629. leaf = path->nodes[0];
  630. item_size = btrfs_item_size_nr(leaf, path->slots[0]);
  631. if (item_size >= sizeof(*ei)) {
  632. ei = btrfs_item_ptr(leaf, path->slots[0],
  633. struct btrfs_extent_item);
  634. num_refs = btrfs_extent_refs(leaf, ei);
  635. extent_flags = btrfs_extent_flags(leaf, ei);
  636. } else {
  637. #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
  638. struct btrfs_extent_item_v0 *ei0;
  639. BUG_ON(item_size != sizeof(*ei0));
  640. ei0 = btrfs_item_ptr(leaf, path->slots[0],
  641. struct btrfs_extent_item_v0);
  642. num_refs = btrfs_extent_refs_v0(leaf, ei0);
  643. /* FIXME: this isn't correct for data */
  644. extent_flags = BTRFS_BLOCK_FLAG_FULL_BACKREF;
  645. #else
  646. BUG();
  647. #endif
  648. }
  649. BUG_ON(num_refs == 0);
  650. } else {
  651. num_refs = 0;
  652. extent_flags = 0;
  653. ret = 0;
  654. }
  655. if (!trans)
  656. goto out;
  657. delayed_refs = &trans->transaction->delayed_refs;
  658. spin_lock(&delayed_refs->lock);
  659. head = btrfs_find_delayed_ref_head(trans, bytenr);
  660. if (head) {
  661. if (!mutex_trylock(&head->mutex)) {
  662. atomic_inc(&head->node.refs);
  663. spin_unlock(&delayed_refs->lock);
  664. btrfs_release_path(root->fs_info->extent_root, path);
  665. mutex_lock(&head->mutex);
  666. mutex_unlock(&head->mutex);
  667. btrfs_put_delayed_ref(&head->node);
  668. goto again;
  669. }
  670. if (head->extent_op && head->extent_op->update_flags)
  671. extent_flags |= head->extent_op->flags_to_set;
  672. else
  673. BUG_ON(num_refs == 0);
  674. num_refs += head->node.ref_mod;
  675. mutex_unlock(&head->mutex);
  676. }
  677. spin_unlock(&delayed_refs->lock);
  678. out:
  679. WARN_ON(num_refs == 0);
  680. if (refs)
  681. *refs = num_refs;
  682. if (flags)
  683. *flags = extent_flags;
  684. out_free:
  685. btrfs_free_path(path);
  686. return ret;
  687. }
  688. /*
  689. * Back reference rules. Back refs have three main goals:
  690. *
  691. * 1) differentiate between all holders of references to an extent so that
  692. * when a reference is dropped we can make sure it was a valid reference
  693. * before freeing the extent.
  694. *
  695. * 2) Provide enough information to quickly find the holders of an extent
  696. * if we notice a given block is corrupted or bad.
  697. *
  698. * 3) Make it easy to migrate blocks for FS shrinking or storage pool
  699. * maintenance. This is actually the same as #2, but with a slightly
  700. * different use case.
  701. *
  702. * There are two kinds of back refs. The implicit back refs is optimized
  703. * for pointers in non-shared tree blocks. For a given pointer in a block,
  704. * back refs of this kind provide information about the block's owner tree
  705. * and the pointer's key. These information allow us to find the block by
  706. * b-tree searching. The full back refs is for pointers in tree blocks not
  707. * referenced by their owner trees. The location of tree block is recorded
  708. * in the back refs. Actually the full back refs is generic, and can be
  709. * used in all cases the implicit back refs is used. The major shortcoming
  710. * of the full back refs is its overhead. Every time a tree block gets
  711. * COWed, we have to update back refs entry for all pointers in it.
  712. *
  713. * For a newly allocated tree block, we use implicit back refs for
  714. * pointers in it. This means most tree related operations only involve
  715. * implicit back refs. For a tree block created in old transaction, the
  716. * only way to drop a reference to it is COW it. So we can detect the
  717. * event that tree block loses its owner tree's reference and do the
  718. * back refs conversion.
  719. *
  720. * When a tree block is COW'd through a tree, there are four cases:
  721. *
  722. * The reference count of the block is one and the tree is the block's
  723. * owner tree. Nothing to do in this case.
  724. *
  725. * The reference count of the block is one and the tree is not the
  726. * block's owner tree. In this case, full back refs is used for pointers
  727. * in the block. Remove these full back refs, add implicit back refs for
  728. * every pointers in the new block.
  729. *
  730. * The reference count of the block is greater than one and the tree is
  731. * the block's owner tree. In this case, implicit back refs is used for
  732. * pointers in the block. Add full back refs for every pointers in the
  733. * block, increase lower level extents' reference counts. The original
  734. * implicit back refs are entailed to the new block.
  735. *
  736. * The reference count of the block is greater than one and the tree is
  737. * not the block's owner tree. Add implicit back refs for every pointer in
  738. * the new block, increase lower level extents' reference count.
  739. *
  740. * Back Reference Key composing:
  741. *
  742. * The key objectid corresponds to the first byte in the extent,
  743. * The key type is used to differentiate between types of back refs.
  744. * There are different meanings of the key offset for different types
  745. * of back refs.
  746. *
  747. * File extents can be referenced by:
  748. *
  749. * - multiple snapshots, subvolumes, or different generations in one subvol
  750. * - different files inside a single subvolume
  751. * - different offsets inside a file (bookend extents in file.c)
  752. *
  753. * The extent ref structure for the implicit back refs has fields for:
  754. *
  755. * - Objectid of the subvolume root
  756. * - objectid of the file holding the reference
  757. * - original offset in the file
  758. * - how many bookend extents
  759. *
  760. * The key offset for the implicit back refs is hash of the first
  761. * three fields.
  762. *
  763. * The extent ref structure for the full back refs has field for:
  764. *
  765. * - number of pointers in the tree leaf
  766. *
  767. * The key offset for the implicit back refs is the first byte of
  768. * the tree leaf
  769. *
  770. * When a file extent is allocated, The implicit back refs is used.
  771. * the fields are filled in:
  772. *
  773. * (root_key.objectid, inode objectid, offset in file, 1)
  774. *
  775. * When a file extent is removed file truncation, we find the
  776. * corresponding implicit back refs and check the following fields:
  777. *
  778. * (btrfs_header_owner(leaf), inode objectid, offset in file)
  779. *
  780. * Btree extents can be referenced by:
  781. *
  782. * - Different subvolumes
  783. *
  784. * Both the implicit back refs and the full back refs for tree blocks
  785. * only consist of key. The key offset for the implicit back refs is
  786. * objectid of block's owner tree. The key offset for the full back refs
  787. * is the first byte of parent block.
  788. *
  789. * When implicit back refs is used, information about the lowest key and
  790. * level of the tree block are required. These information are stored in
  791. * tree block info structure.
  792. */
  793. #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
  794. static int convert_extent_item_v0(struct btrfs_trans_handle *trans,
  795. struct btrfs_root *root,
  796. struct btrfs_path *path,
  797. u64 owner, u32 extra_size)
  798. {
  799. struct btrfs_extent_item *item;
  800. struct btrfs_extent_item_v0 *ei0;
  801. struct btrfs_extent_ref_v0 *ref0;
  802. struct btrfs_tree_block_info *bi;
  803. struct extent_buffer *leaf;
  804. struct btrfs_key key;
  805. struct btrfs_key found_key;
  806. u32 new_size = sizeof(*item);
  807. u64 refs;
  808. int ret;
  809. leaf = path->nodes[0];
  810. BUG_ON(btrfs_item_size_nr(leaf, path->slots[0]) != sizeof(*ei0));
  811. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  812. ei0 = btrfs_item_ptr(leaf, path->slots[0],
  813. struct btrfs_extent_item_v0);
  814. refs = btrfs_extent_refs_v0(leaf, ei0);
  815. if (owner == (u64)-1) {
  816. while (1) {
  817. if (path->slots[0] >= btrfs_header_nritems(leaf)) {
  818. ret = btrfs_next_leaf(root, path);
  819. if (ret < 0)
  820. return ret;
  821. BUG_ON(ret > 0);
  822. leaf = path->nodes[0];
  823. }
  824. btrfs_item_key_to_cpu(leaf, &found_key,
  825. path->slots[0]);
  826. BUG_ON(key.objectid != found_key.objectid);
  827. if (found_key.type != BTRFS_EXTENT_REF_V0_KEY) {
  828. path->slots[0]++;
  829. continue;
  830. }
  831. ref0 = btrfs_item_ptr(leaf, path->slots[0],
  832. struct btrfs_extent_ref_v0);
  833. owner = btrfs_ref_objectid_v0(leaf, ref0);
  834. break;
  835. }
  836. }
  837. btrfs_release_path(root, path);
  838. if (owner < BTRFS_FIRST_FREE_OBJECTID)
  839. new_size += sizeof(*bi);
  840. new_size -= sizeof(*ei0);
  841. ret = btrfs_search_slot(trans, root, &key, path,
  842. new_size + extra_size, 1);
  843. if (ret < 0)
  844. return ret;
  845. BUG_ON(ret);
  846. ret = btrfs_extend_item(trans, root, path, new_size);
  847. BUG_ON(ret);
  848. leaf = path->nodes[0];
  849. item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
  850. btrfs_set_extent_refs(leaf, item, refs);
  851. /* FIXME: get real generation */
  852. btrfs_set_extent_generation(leaf, item, 0);
  853. if (owner < BTRFS_FIRST_FREE_OBJECTID) {
  854. btrfs_set_extent_flags(leaf, item,
  855. BTRFS_EXTENT_FLAG_TREE_BLOCK |
  856. BTRFS_BLOCK_FLAG_FULL_BACKREF);
  857. bi = (struct btrfs_tree_block_info *)(item + 1);
  858. /* FIXME: get first key of the block */
  859. memset_extent_buffer(leaf, 0, (unsigned long)bi, sizeof(*bi));
  860. btrfs_set_tree_block_level(leaf, bi, (int)owner);
  861. } else {
  862. btrfs_set_extent_flags(leaf, item, BTRFS_EXTENT_FLAG_DATA);
  863. }
  864. btrfs_mark_buffer_dirty(leaf);
  865. return 0;
  866. }
  867. #endif
  868. static u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset)
  869. {
  870. u32 high_crc = ~(u32)0;
  871. u32 low_crc = ~(u32)0;
  872. __le64 lenum;
  873. lenum = cpu_to_le64(root_objectid);
  874. high_crc = crc32c(high_crc, &lenum, sizeof(lenum));
  875. lenum = cpu_to_le64(owner);
  876. low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
  877. lenum = cpu_to_le64(offset);
  878. low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
  879. return ((u64)high_crc << 31) ^ (u64)low_crc;
  880. }
  881. static u64 hash_extent_data_ref_item(struct extent_buffer *leaf,
  882. struct btrfs_extent_data_ref *ref)
  883. {
  884. return hash_extent_data_ref(btrfs_extent_data_ref_root(leaf, ref),
  885. btrfs_extent_data_ref_objectid(leaf, ref),
  886. btrfs_extent_data_ref_offset(leaf, ref));
  887. }
  888. static int match_extent_data_ref(struct extent_buffer *leaf,
  889. struct btrfs_extent_data_ref *ref,
  890. u64 root_objectid, u64 owner, u64 offset)
  891. {
  892. if (btrfs_extent_data_ref_root(leaf, ref) != root_objectid ||
  893. btrfs_extent_data_ref_objectid(leaf, ref) != owner ||
  894. btrfs_extent_data_ref_offset(leaf, ref) != offset)
  895. return 0;
  896. return 1;
  897. }
  898. static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans,
  899. struct btrfs_root *root,
  900. struct btrfs_path *path,
  901. u64 bytenr, u64 parent,
  902. u64 root_objectid,
  903. u64 owner, u64 offset)
  904. {
  905. struct btrfs_key key;
  906. struct btrfs_extent_data_ref *ref;
  907. struct extent_buffer *leaf;
  908. u32 nritems;
  909. int ret;
  910. int recow;
  911. int err = -ENOENT;
  912. key.objectid = bytenr;
  913. if (parent) {
  914. key.type = BTRFS_SHARED_DATA_REF_KEY;
  915. key.offset = parent;
  916. } else {
  917. key.type = BTRFS_EXTENT_DATA_REF_KEY;
  918. key.offset = hash_extent_data_ref(root_objectid,
  919. owner, offset);
  920. }
  921. again:
  922. recow = 0;
  923. ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
  924. if (ret < 0) {
  925. err = ret;
  926. goto fail;
  927. }
  928. if (parent) {
  929. if (!ret)
  930. return 0;
  931. #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
  932. key.type = BTRFS_EXTENT_REF_V0_KEY;
  933. btrfs_release_path(root, path);
  934. ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
  935. if (ret < 0) {
  936. err = ret;
  937. goto fail;
  938. }
  939. if (!ret)
  940. return 0;
  941. #endif
  942. goto fail;
  943. }
  944. leaf = path->nodes[0];
  945. nritems = btrfs_header_nritems(leaf);
  946. while (1) {
  947. if (path->slots[0] >= nritems) {
  948. ret = btrfs_next_leaf(root, path);
  949. if (ret < 0)
  950. err = ret;
  951. if (ret)
  952. goto fail;
  953. leaf = path->nodes[0];
  954. nritems = btrfs_header_nritems(leaf);
  955. recow = 1;
  956. }
  957. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  958. if (key.objectid != bytenr ||
  959. key.type != BTRFS_EXTENT_DATA_REF_KEY)
  960. goto fail;
  961. ref = btrfs_item_ptr(leaf, path->slots[0],
  962. struct btrfs_extent_data_ref);
  963. if (match_extent_data_ref(leaf, ref, root_objectid,
  964. owner, offset)) {
  965. if (recow) {
  966. btrfs_release_path(root, path);
  967. goto again;
  968. }
  969. err = 0;
  970. break;
  971. }
  972. path->slots[0]++;
  973. }
  974. fail:
  975. return err;
  976. }
  977. static noinline int insert_extent_data_ref(struct btrfs_trans_handle *trans,
  978. struct btrfs_root *root,
  979. struct btrfs_path *path,
  980. u64 bytenr, u64 parent,
  981. u64 root_objectid, u64 owner,
  982. u64 offset, int refs_to_add)
  983. {
  984. struct btrfs_key key;
  985. struct extent_buffer *leaf;
  986. u32 size;
  987. u32 num_refs;
  988. int ret;
  989. key.objectid = bytenr;
  990. if (parent) {
  991. key.type = BTRFS_SHARED_DATA_REF_KEY;
  992. key.offset = parent;
  993. size = sizeof(struct btrfs_shared_data_ref);
  994. } else {
  995. key.type = BTRFS_EXTENT_DATA_REF_KEY;
  996. key.offset = hash_extent_data_ref(root_objectid,
  997. owner, offset);
  998. size = sizeof(struct btrfs_extent_data_ref);
  999. }
  1000. ret = btrfs_insert_empty_item(trans, root, path, &key, size);
  1001. if (ret && ret != -EEXIST)
  1002. goto fail;
  1003. leaf = path->nodes[0];
  1004. if (parent) {
  1005. struct btrfs_shared_data_ref *ref;
  1006. ref = btrfs_item_ptr(leaf, path->slots[0],
  1007. struct btrfs_shared_data_ref);
  1008. if (ret == 0) {
  1009. btrfs_set_shared_data_ref_count(leaf, ref, refs_to_add);
  1010. } else {
  1011. num_refs = btrfs_shared_data_ref_count(leaf, ref);
  1012. num_refs += refs_to_add;
  1013. btrfs_set_shared_data_ref_count(leaf, ref, num_refs);
  1014. }
  1015. } else {
  1016. struct btrfs_extent_data_ref *ref;
  1017. while (ret == -EEXIST) {
  1018. ref = btrfs_item_ptr(leaf, path->slots[0],
  1019. struct btrfs_extent_data_ref);
  1020. if (match_extent_data_ref(leaf, ref, root_objectid,
  1021. owner, offset))
  1022. break;
  1023. btrfs_release_path(root, path);
  1024. key.offset++;
  1025. ret = btrfs_insert_empty_item(trans, root, path, &key,
  1026. size);
  1027. if (ret && ret != -EEXIST)
  1028. goto fail;
  1029. leaf = path->nodes[0];
  1030. }
  1031. ref = btrfs_item_ptr(leaf, path->slots[0],
  1032. struct btrfs_extent_data_ref);
  1033. if (ret == 0) {
  1034. btrfs_set_extent_data_ref_root(leaf, ref,
  1035. root_objectid);
  1036. btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
  1037. btrfs_set_extent_data_ref_offset(leaf, ref, offset);
  1038. btrfs_set_extent_data_ref_count(leaf, ref, refs_to_add);
  1039. } else {
  1040. num_refs = btrfs_extent_data_ref_count(leaf, ref);
  1041. num_refs += refs_to_add;
  1042. btrfs_set_extent_data_ref_count(leaf, ref, num_refs);
  1043. }
  1044. }
  1045. btrfs_mark_buffer_dirty(leaf);
  1046. ret = 0;
  1047. fail:
  1048. btrfs_release_path(root, path);
  1049. return ret;
  1050. }
  1051. static noinline int remove_extent_data_ref(struct btrfs_trans_handle *trans,
  1052. struct btrfs_root *root,
  1053. struct btrfs_path *path,
  1054. int refs_to_drop)
  1055. {
  1056. struct btrfs_key key;
  1057. struct btrfs_extent_data_ref *ref1 = NULL;
  1058. struct btrfs_shared_data_ref *ref2 = NULL;
  1059. struct extent_buffer *leaf;
  1060. u32 num_refs = 0;
  1061. int ret = 0;
  1062. leaf = path->nodes[0];
  1063. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  1064. if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
  1065. ref1 = btrfs_item_ptr(leaf, path->slots[0],
  1066. struct btrfs_extent_data_ref);
  1067. num_refs = btrfs_extent_data_ref_count(leaf, ref1);
  1068. } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
  1069. ref2 = btrfs_item_ptr(leaf, path->slots[0],
  1070. struct btrfs_shared_data_ref);
  1071. num_refs = btrfs_shared_data_ref_count(leaf, ref2);
  1072. #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
  1073. } else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
  1074. struct btrfs_extent_ref_v0 *ref0;
  1075. ref0 = btrfs_item_ptr(leaf, path->slots[0],
  1076. struct btrfs_extent_ref_v0);
  1077. num_refs = btrfs_ref_count_v0(leaf, ref0);
  1078. #endif
  1079. } else {
  1080. BUG();
  1081. }
  1082. BUG_ON(num_refs < refs_to_drop);
  1083. num_refs -= refs_to_drop;
  1084. if (num_refs == 0) {
  1085. ret = btrfs_del_item(trans, root, path);
  1086. } else {
  1087. if (key.type == BTRFS_EXTENT_DATA_REF_KEY)
  1088. btrfs_set_extent_data_ref_count(leaf, ref1, num_refs);
  1089. else if (key.type == BTRFS_SHARED_DATA_REF_KEY)
  1090. btrfs_set_shared_data_ref_count(leaf, ref2, num_refs);
  1091. #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
  1092. else {
  1093. struct btrfs_extent_ref_v0 *ref0;
  1094. ref0 = btrfs_item_ptr(leaf, path->slots[0],
  1095. struct btrfs_extent_ref_v0);
  1096. btrfs_set_ref_count_v0(leaf, ref0, num_refs);
  1097. }
  1098. #endif
  1099. btrfs_mark_buffer_dirty(leaf);
  1100. }
  1101. return ret;
  1102. }
  1103. static noinline u32 extent_data_ref_count(struct btrfs_root *root,
  1104. struct btrfs_path *path,
  1105. struct btrfs_extent_inline_ref *iref)
  1106. {
  1107. struct btrfs_key key;
  1108. struct extent_buffer *leaf;
  1109. struct btrfs_extent_data_ref *ref1;
  1110. struct btrfs_shared_data_ref *ref2;
  1111. u32 num_refs = 0;
  1112. leaf = path->nodes[0];
  1113. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  1114. if (iref) {
  1115. if (btrfs_extent_inline_ref_type(leaf, iref) ==
  1116. BTRFS_EXTENT_DATA_REF_KEY) {
  1117. ref1 = (struct btrfs_extent_data_ref *)(&iref->offset);
  1118. num_refs = btrfs_extent_data_ref_count(leaf, ref1);
  1119. } else {
  1120. ref2 = (struct btrfs_shared_data_ref *)(iref + 1);
  1121. num_refs = btrfs_shared_data_ref_count(leaf, ref2);
  1122. }
  1123. } else if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
  1124. ref1 = btrfs_item_ptr(leaf, path->slots[0],
  1125. struct btrfs_extent_data_ref);
  1126. num_refs = btrfs_extent_data_ref_count(leaf, ref1);
  1127. } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
  1128. ref2 = btrfs_item_ptr(leaf, path->slots[0],
  1129. struct btrfs_shared_data_ref);
  1130. num_refs = btrfs_shared_data_ref_count(leaf, ref2);
  1131. #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
  1132. } else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
  1133. struct btrfs_extent_ref_v0 *ref0;
  1134. ref0 = btrfs_item_ptr(leaf, path->slots[0],
  1135. struct btrfs_extent_ref_v0);
  1136. num_refs = btrfs_ref_count_v0(leaf, ref0);
  1137. #endif
  1138. } else {
  1139. WARN_ON(1);
  1140. }
  1141. return num_refs;
  1142. }
  1143. static noinline int lookup_tree_block_ref(struct btrfs_trans_handle *trans,
  1144. struct btrfs_root *root,
  1145. struct btrfs_path *path,
  1146. u64 bytenr, u64 parent,
  1147. u64 root_objectid)
  1148. {
  1149. struct btrfs_key key;
  1150. int ret;
  1151. key.objectid = bytenr;
  1152. if (parent) {
  1153. key.type = BTRFS_SHARED_BLOCK_REF_KEY;
  1154. key.offset = parent;
  1155. } else {
  1156. key.type = BTRFS_TREE_BLOCK_REF_KEY;
  1157. key.offset = root_objectid;
  1158. }
  1159. ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
  1160. if (ret > 0)
  1161. ret = -ENOENT;
  1162. #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
  1163. if (ret == -ENOENT && parent) {
  1164. btrfs_release_path(root, path);
  1165. key.type = BTRFS_EXTENT_REF_V0_KEY;
  1166. ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
  1167. if (ret > 0)
  1168. ret = -ENOENT;
  1169. }
  1170. #endif
  1171. return ret;
  1172. }
  1173. static noinline int insert_tree_block_ref(struct btrfs_trans_handle *trans,
  1174. struct btrfs_root *root,
  1175. struct btrfs_path *path,
  1176. u64 bytenr, u64 parent,
  1177. u64 root_objectid)
  1178. {
  1179. struct btrfs_key key;
  1180. int ret;
  1181. key.objectid = bytenr;
  1182. if (parent) {
  1183. key.type = BTRFS_SHARED_BLOCK_REF_KEY;
  1184. key.offset = parent;
  1185. } else {
  1186. key.type = BTRFS_TREE_BLOCK_REF_KEY;
  1187. key.offset = root_objectid;
  1188. }
  1189. ret = btrfs_insert_empty_item(trans, root, path, &key, 0);
  1190. btrfs_release_path(root, path);
  1191. return ret;
  1192. }
  1193. static inline int extent_ref_type(u64 parent, u64 owner)
  1194. {
  1195. int type;
  1196. if (owner < BTRFS_FIRST_FREE_OBJECTID) {
  1197. if (parent > 0)
  1198. type = BTRFS_SHARED_BLOCK_REF_KEY;
  1199. else
  1200. type = BTRFS_TREE_BLOCK_REF_KEY;
  1201. } else {
  1202. if (parent > 0)
  1203. type = BTRFS_SHARED_DATA_REF_KEY;
  1204. else
  1205. type = BTRFS_EXTENT_DATA_REF_KEY;
  1206. }
  1207. return type;
  1208. }
  1209. static int find_next_key(struct btrfs_path *path, int level,
  1210. struct btrfs_key *key)
  1211. {
  1212. for (; level < BTRFS_MAX_LEVEL; level++) {
  1213. if (!path->nodes[level])
  1214. break;
  1215. if (path->slots[level] + 1 >=
  1216. btrfs_header_nritems(path->nodes[level]))
  1217. continue;
  1218. if (level == 0)
  1219. btrfs_item_key_to_cpu(path->nodes[level], key,
  1220. path->slots[level] + 1);
  1221. else
  1222. btrfs_node_key_to_cpu(path->nodes[level], key,
  1223. path->slots[level] + 1);
  1224. return 0;
  1225. }
  1226. return 1;
  1227. }
  1228. /*
  1229. * look for inline back ref. if back ref is found, *ref_ret is set
  1230. * to the address of inline back ref, and 0 is returned.
  1231. *
  1232. * if back ref isn't found, *ref_ret is set to the address where it
  1233. * should be inserted, and -ENOENT is returned.
  1234. *
  1235. * if insert is true and there are too many inline back refs, the path
  1236. * points to the extent item, and -EAGAIN is returned.
  1237. *
  1238. * NOTE: inline back refs are ordered in the same way that back ref
  1239. * items in the tree are ordered.
  1240. */
  1241. static noinline_for_stack
  1242. int lookup_inline_extent_backref(struct btrfs_trans_handle *trans,
  1243. struct btrfs_root *root,
  1244. struct btrfs_path *path,
  1245. struct btrfs_extent_inline_ref **ref_ret,
  1246. u64 bytenr, u64 num_bytes,
  1247. u64 parent, u64 root_objectid,
  1248. u64 owner, u64 offset, int insert)
  1249. {
  1250. struct btrfs_key key;
  1251. struct extent_buffer *leaf;
  1252. struct btrfs_extent_item *ei;
  1253. struct btrfs_extent_inline_ref *iref;
  1254. u64 flags;
  1255. u64 item_size;
  1256. unsigned long ptr;
  1257. unsigned long end;
  1258. int extra_size;
  1259. int type;
  1260. int want;
  1261. int ret;
  1262. int err = 0;
  1263. key.objectid = bytenr;
  1264. key.type = BTRFS_EXTENT_ITEM_KEY;
  1265. key.offset = num_bytes;
  1266. want = extent_ref_type(parent, owner);
  1267. if (insert) {
  1268. extra_size = btrfs_extent_inline_ref_size(want);
  1269. path->keep_locks = 1;
  1270. } else
  1271. extra_size = -1;
  1272. ret = btrfs_search_slot(trans, root, &key, path, extra_size, 1);
  1273. if (ret < 0) {
  1274. err = ret;
  1275. goto out;
  1276. }
  1277. BUG_ON(ret);
  1278. leaf = path->nodes[0];
  1279. item_size = btrfs_item_size_nr(leaf, path->slots[0]);
  1280. #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
  1281. if (item_size < sizeof(*ei)) {
  1282. if (!insert) {
  1283. err = -ENOENT;
  1284. goto out;
  1285. }
  1286. ret = convert_extent_item_v0(trans, root, path, owner,
  1287. extra_size);
  1288. if (ret < 0) {
  1289. err = ret;
  1290. goto out;
  1291. }
  1292. leaf = path->nodes[0];
  1293. item_size = btrfs_item_size_nr(leaf, path->slots[0]);
  1294. }
  1295. #endif
  1296. BUG_ON(item_size < sizeof(*ei));
  1297. ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
  1298. flags = btrfs_extent_flags(leaf, ei);
  1299. ptr = (unsigned long)(ei + 1);
  1300. end = (unsigned long)ei + item_size;
  1301. if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
  1302. ptr += sizeof(struct btrfs_tree_block_info);
  1303. BUG_ON(ptr > end);
  1304. } else {
  1305. BUG_ON(!(flags & BTRFS_EXTENT_FLAG_DATA));
  1306. }
  1307. err = -ENOENT;
  1308. while (1) {
  1309. if (ptr >= end) {
  1310. WARN_ON(ptr > end);
  1311. break;
  1312. }
  1313. iref = (struct btrfs_extent_inline_ref *)ptr;
  1314. type = btrfs_extent_inline_ref_type(leaf, iref);
  1315. if (want < type)
  1316. break;
  1317. if (want > type) {
  1318. ptr += btrfs_extent_inline_ref_size(type);
  1319. continue;
  1320. }
  1321. if (type == BTRFS_EXTENT_DATA_REF_KEY) {
  1322. struct btrfs_extent_data_ref *dref;
  1323. dref = (struct btrfs_extent_data_ref *)(&iref->offset);
  1324. if (match_extent_data_ref(leaf, dref, root_objectid,
  1325. owner, offset)) {
  1326. err = 0;
  1327. break;
  1328. }
  1329. if (hash_extent_data_ref_item(leaf, dref) <
  1330. hash_extent_data_ref(root_objectid, owner, offset))
  1331. break;
  1332. } else {
  1333. u64 ref_offset;
  1334. ref_offset = btrfs_extent_inline_ref_offset(leaf, iref);
  1335. if (parent > 0) {
  1336. if (parent == ref_offset) {
  1337. err = 0;
  1338. break;
  1339. }
  1340. if (ref_offset < parent)
  1341. break;
  1342. } else {
  1343. if (root_objectid == ref_offset) {
  1344. err = 0;
  1345. break;
  1346. }
  1347. if (ref_offset < root_objectid)
  1348. break;
  1349. }
  1350. }
  1351. ptr += btrfs_extent_inline_ref_size(type);
  1352. }
  1353. if (err == -ENOENT && insert) {
  1354. if (item_size + extra_size >=
  1355. BTRFS_MAX_EXTENT_ITEM_SIZE(root)) {
  1356. err = -EAGAIN;
  1357. goto out;
  1358. }
  1359. /*
  1360. * To add new inline back ref, we have to make sure
  1361. * there is no corresponding back ref item.
  1362. * For simplicity, we just do not add new inline back
  1363. * ref if there is any kind of item for this block
  1364. */
  1365. if (find_next_key(path, 0, &key) == 0 &&
  1366. key.objectid == bytenr &&
  1367. key.type < BTRFS_BLOCK_GROUP_ITEM_KEY) {
  1368. err = -EAGAIN;
  1369. goto out;
  1370. }
  1371. }
  1372. *ref_ret = (struct btrfs_extent_inline_ref *)ptr;
  1373. out:
  1374. if (insert) {
  1375. path->keep_locks = 0;
  1376. btrfs_unlock_up_safe(path, 1);
  1377. }
  1378. return err;
  1379. }
  1380. /*
  1381. * helper to add new inline back ref
  1382. */
  1383. static noinline_for_stack
  1384. int setup_inline_extent_backref(struct btrfs_trans_handle *trans,
  1385. struct btrfs_root *root,
  1386. struct btrfs_path *path,
  1387. struct btrfs_extent_inline_ref *iref,
  1388. u64 parent, u64 root_objectid,
  1389. u64 owner, u64 offset, int refs_to_add,
  1390. struct btrfs_delayed_extent_op *extent_op)
  1391. {
  1392. struct extent_buffer *leaf;
  1393. struct btrfs_extent_item *ei;
  1394. unsigned long ptr;
  1395. unsigned long end;
  1396. unsigned long item_offset;
  1397. u64 refs;
  1398. int size;
  1399. int type;
  1400. int ret;
  1401. leaf = path->nodes[0];
  1402. ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
  1403. item_offset = (unsigned long)iref - (unsigned long)ei;
  1404. type = extent_ref_type(parent, owner);
  1405. size = btrfs_extent_inline_ref_size(type);
  1406. ret = btrfs_extend_item(trans, root, path, size);
  1407. BUG_ON(ret);
  1408. ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
  1409. refs = btrfs_extent_refs(leaf, ei);
  1410. refs += refs_to_add;
  1411. btrfs_set_extent_refs(leaf, ei, refs);
  1412. if (extent_op)
  1413. __run_delayed_extent_op(extent_op, leaf, ei);
  1414. ptr = (unsigned long)ei + item_offset;
  1415. end = (unsigned long)ei + btrfs_item_size_nr(leaf, path->slots[0]);
  1416. if (ptr < end - size)
  1417. memmove_extent_buffer(leaf, ptr + size, ptr,
  1418. end - size - ptr);
  1419. iref = (struct btrfs_extent_inline_ref *)ptr;
  1420. btrfs_set_extent_inline_ref_type(leaf, iref, type);
  1421. if (type == BTRFS_EXTENT_DATA_REF_KEY) {
  1422. struct btrfs_extent_data_ref *dref;
  1423. dref = (struct btrfs_extent_data_ref *)(&iref->offset);
  1424. btrfs_set_extent_data_ref_root(leaf, dref, root_objectid);
  1425. btrfs_set_extent_data_ref_objectid(leaf, dref, owner);
  1426. btrfs_set_extent_data_ref_offset(leaf, dref, offset);
  1427. btrfs_set_extent_data_ref_count(leaf, dref, refs_to_add);
  1428. } else if (type == BTRFS_SHARED_DATA_REF_KEY) {
  1429. struct btrfs_shared_data_ref *sref;
  1430. sref = (struct btrfs_shared_data_ref *)(iref + 1);
  1431. btrfs_set_shared_data_ref_count(leaf, sref, refs_to_add);
  1432. btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
  1433. } else if (type == BTRFS_SHARED_BLOCK_REF_KEY) {
  1434. btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
  1435. } else {
  1436. btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
  1437. }
  1438. btrfs_mark_buffer_dirty(leaf);
  1439. return 0;
  1440. }
  1441. static int lookup_extent_backref(struct btrfs_trans_handle *trans,
  1442. struct btrfs_root *root,
  1443. struct btrfs_path *path,
  1444. struct btrfs_extent_inline_ref **ref_ret,
  1445. u64 bytenr, u64 num_bytes, u64 parent,
  1446. u64 root_objectid, u64 owner, u64 offset)
  1447. {
  1448. int ret;
  1449. ret = lookup_inline_extent_backref(trans, root, path, ref_ret,
  1450. bytenr, num_bytes, parent,
  1451. root_objectid, owner, offset, 0);
  1452. if (ret != -ENOENT)
  1453. return ret;
  1454. btrfs_release_path(root, path);
  1455. *ref_ret = NULL;
  1456. if (owner < BTRFS_FIRST_FREE_OBJECTID) {
  1457. ret = lookup_tree_block_ref(trans, root, path, bytenr, parent,
  1458. root_objectid);
  1459. } else {
  1460. ret = lookup_extent_data_ref(trans, root, path, bytenr, parent,
  1461. root_objectid, owner, offset);
  1462. }
  1463. return ret;
  1464. }
  1465. /*
  1466. * helper to update/remove inline back ref
  1467. */
  1468. static noinline_for_stack
  1469. int update_inline_extent_backref(struct btrfs_trans_handle *trans,
  1470. struct btrfs_root *root,
  1471. struct btrfs_path *path,
  1472. struct btrfs_extent_inline_ref *iref,
  1473. int refs_to_mod,
  1474. struct btrfs_delayed_extent_op *extent_op)
  1475. {
  1476. struct extent_buffer *leaf;
  1477. struct btrfs_extent_item *ei;
  1478. struct btrfs_extent_data_ref *dref = NULL;
  1479. struct btrfs_shared_data_ref *sref = NULL;
  1480. unsigned long ptr;
  1481. unsigned long end;
  1482. u32 item_size;
  1483. int size;
  1484. int type;
  1485. int ret;
  1486. u64 refs;
  1487. leaf = path->nodes[0];
  1488. ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
  1489. refs = btrfs_extent_refs(leaf, ei);
  1490. WARN_ON(refs_to_mod < 0 && refs + refs_to_mod <= 0);
  1491. refs += refs_to_mod;
  1492. btrfs_set_extent_refs(leaf, ei, refs);
  1493. if (extent_op)
  1494. __run_delayed_extent_op(extent_op, leaf, ei);
  1495. type = btrfs_extent_inline_ref_type(leaf, iref);
  1496. if (type == BTRFS_EXTENT_DATA_REF_KEY) {
  1497. dref = (struct btrfs_extent_data_ref *)(&iref->offset);
  1498. refs = btrfs_extent_data_ref_count(leaf, dref);
  1499. } else if (type == BTRFS_SHARED_DATA_REF_KEY) {
  1500. sref = (struct btrfs_shared_data_ref *)(iref + 1);
  1501. refs = btrfs_shared_data_ref_count(leaf, sref);
  1502. } else {
  1503. refs = 1;
  1504. BUG_ON(refs_to_mod != -1);
  1505. }
  1506. BUG_ON(refs_to_mod < 0 && refs < -refs_to_mod);
  1507. refs += refs_to_mod;
  1508. if (refs > 0) {
  1509. if (type == BTRFS_EXTENT_DATA_REF_KEY)
  1510. btrfs_set_extent_data_ref_count(leaf, dref, refs);
  1511. else
  1512. btrfs_set_shared_data_ref_count(leaf, sref, refs);
  1513. } else {
  1514. size = btrfs_extent_inline_ref_size(type);
  1515. item_size = btrfs_item_size_nr(leaf, path->slots[0]);
  1516. ptr = (unsigned long)iref;
  1517. end = (unsigned long)ei + item_size;
  1518. if (ptr + size < end)
  1519. memmove_extent_buffer(leaf, ptr, ptr + size,
  1520. end - ptr - size);
  1521. item_size -= size;
  1522. ret = btrfs_truncate_item(trans, root, path, item_size, 1);
  1523. BUG_ON(ret);
  1524. }
  1525. btrfs_mark_buffer_dirty(leaf);
  1526. return 0;
  1527. }
  1528. static noinline_for_stack
  1529. int insert_inline_extent_backref(struct btrfs_trans_handle *trans,
  1530. struct btrfs_root *root,
  1531. struct btrfs_path *path,
  1532. u64 bytenr, u64 num_bytes, u64 parent,
  1533. u64 root_objectid, u64 owner,
  1534. u64 offset, int refs_to_add,
  1535. struct btrfs_delayed_extent_op *extent_op)
  1536. {
  1537. struct btrfs_extent_inline_ref *iref;
  1538. int ret;
  1539. ret = lookup_inline_extent_backref(trans, root, path, &iref,
  1540. bytenr, num_bytes, parent,
  1541. root_objectid, owner, offset, 1);
  1542. if (ret == 0) {
  1543. BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID);
  1544. ret = update_inline_extent_backref(trans, root, path, iref,
  1545. refs_to_add, extent_op);
  1546. } else if (ret == -ENOENT) {
  1547. ret = setup_inline_extent_backref(trans, root, path, iref,
  1548. parent, root_objectid,
  1549. owner, offset, refs_to_add,
  1550. extent_op);
  1551. }
  1552. return ret;
  1553. }
  1554. static int insert_extent_backref(struct btrfs_trans_handle *trans,
  1555. struct btrfs_root *root,
  1556. struct btrfs_path *path,
  1557. u64 bytenr, u64 parent, u64 root_objectid,
  1558. u64 owner, u64 offset, int refs_to_add)
  1559. {
  1560. int ret;
  1561. if (owner < BTRFS_FIRST_FREE_OBJECTID) {
  1562. BUG_ON(refs_to_add != 1);
  1563. ret = insert_tree_block_ref(trans, root, path, bytenr,
  1564. parent, root_objectid);
  1565. } else {
  1566. ret = insert_extent_data_ref(trans, root, path, bytenr,
  1567. parent, root_objectid,
  1568. owner, offset, refs_to_add);
  1569. }
  1570. return ret;
  1571. }
  1572. static int remove_extent_backref(struct btrfs_trans_handle *trans,
  1573. struct btrfs_root *root,
  1574. struct btrfs_path *path,
  1575. struct btrfs_extent_inline_ref *iref,
  1576. int refs_to_drop, int is_data)
  1577. {
  1578. int ret;
  1579. BUG_ON(!is_data && refs_to_drop != 1);
  1580. if (iref) {
  1581. ret = update_inline_extent_backref(trans, root, path, iref,
  1582. -refs_to_drop, NULL);
  1583. } else if (is_data) {
  1584. ret = remove_extent_data_ref(trans, root, path, refs_to_drop);
  1585. } else {
  1586. ret = btrfs_del_item(trans, root, path);
  1587. }
  1588. return ret;
  1589. }
  1590. static int btrfs_issue_discard(struct block_device *bdev,
  1591. u64 start, u64 len)
  1592. {
  1593. return blkdev_issue_discard(bdev, start >> 9, len >> 9, GFP_NOFS, 0);
  1594. }
  1595. static int btrfs_discard_extent(struct btrfs_root *root, u64 bytenr,
  1596. u64 num_bytes, u64 *actual_bytes)
  1597. {
  1598. int ret;
  1599. u64 discarded_bytes = 0;
  1600. struct btrfs_multi_bio *multi = NULL;
  1601. /* Tell the block device(s) that the sectors can be discarded */
  1602. ret = btrfs_map_block(&root->fs_info->mapping_tree, REQ_DISCARD,
  1603. bytenr, &num_bytes, &multi, 0);
  1604. if (!ret) {
  1605. struct btrfs_bio_stripe *stripe = multi->stripes;
  1606. int i;
  1607. for (i = 0; i < multi->num_stripes; i++, stripe++) {
  1608. ret = btrfs_issue_discard(stripe->dev->bdev,
  1609. stripe->physical,
  1610. stripe->length);
  1611. if (!ret)
  1612. discarded_bytes += stripe->length;
  1613. else if (ret != -EOPNOTSUPP)
  1614. break;
  1615. }
  1616. kfree(multi);
  1617. }
  1618. if (discarded_bytes && ret == -EOPNOTSUPP)
  1619. ret = 0;
  1620. if (actual_bytes)
  1621. *actual_bytes = discarded_bytes;
  1622. return ret;
  1623. }
  1624. int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
  1625. struct btrfs_root *root,
  1626. u64 bytenr, u64 num_bytes, u64 parent,
  1627. u64 root_objectid, u64 owner, u64 offset)
  1628. {
  1629. int ret;
  1630. BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID &&
  1631. root_objectid == BTRFS_TREE_LOG_OBJECTID);
  1632. if (owner < BTRFS_FIRST_FREE_OBJECTID) {
  1633. ret = btrfs_add_delayed_tree_ref(trans, bytenr, num_bytes,
  1634. parent, root_objectid, (int)owner,
  1635. BTRFS_ADD_DELAYED_REF, NULL);
  1636. } else {
  1637. ret = btrfs_add_delayed_data_ref(trans, bytenr, num_bytes,
  1638. parent, root_objectid, owner, offset,
  1639. BTRFS_ADD_DELAYED_REF, NULL);
  1640. }
  1641. return ret;
  1642. }
  1643. static int __btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
  1644. struct btrfs_root *root,
  1645. u64 bytenr, u64 num_bytes,
  1646. u64 parent, u64 root_objectid,
  1647. u64 owner, u64 offset, int refs_to_add,
  1648. struct btrfs_delayed_extent_op *extent_op)
  1649. {
  1650. struct btrfs_path *path;
  1651. struct extent_buffer *leaf;
  1652. struct btrfs_extent_item *item;
  1653. u64 refs;
  1654. int ret;
  1655. int err = 0;
  1656. path = btrfs_alloc_path();
  1657. if (!path)
  1658. return -ENOMEM;
  1659. path->reada = 1;
  1660. path->leave_spinning = 1;
  1661. /* this will setup the path even if it fails to insert the back ref */
  1662. ret = insert_inline_extent_backref(trans, root->fs_info->extent_root,
  1663. path, bytenr, num_bytes, parent,
  1664. root_objectid, owner, offset,
  1665. refs_to_add, extent_op);
  1666. if (ret == 0)
  1667. goto out;
  1668. if (ret != -EAGAIN) {
  1669. err = ret;
  1670. goto out;
  1671. }
  1672. leaf = path->nodes[0];
  1673. item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
  1674. refs = btrfs_extent_refs(leaf, item);
  1675. btrfs_set_extent_refs(leaf, item, refs + refs_to_add);
  1676. if (extent_op)
  1677. __run_delayed_extent_op(extent_op, leaf, item);
  1678. btrfs_mark_buffer_dirty(leaf);
  1679. btrfs_release_path(root->fs_info->extent_root, path);
  1680. path->reada = 1;
  1681. path->leave_spinning = 1;
  1682. /* now insert the actual backref */
  1683. ret = insert_extent_backref(trans, root->fs_info->extent_root,
  1684. path, bytenr, parent, root_objectid,
  1685. owner, offset, refs_to_add);
  1686. BUG_ON(ret);
  1687. out:
  1688. btrfs_free_path(path);
  1689. return err;
  1690. }
  1691. static int run_delayed_data_ref(struct btrfs_trans_handle *trans,
  1692. struct btrfs_root *root,
  1693. struct btrfs_delayed_ref_node *node,
  1694. struct btrfs_delayed_extent_op *extent_op,
  1695. int insert_reserved)
  1696. {
  1697. int ret = 0;
  1698. struct btrfs_delayed_data_ref *ref;
  1699. struct btrfs_key ins;
  1700. u64 parent = 0;
  1701. u64 ref_root = 0;
  1702. u64 flags = 0;
  1703. ins.objectid = node->bytenr;
  1704. ins.offset = node->num_bytes;
  1705. ins.type = BTRFS_EXTENT_ITEM_KEY;
  1706. ref = btrfs_delayed_node_to_data_ref(node);
  1707. if (node->type == BTRFS_SHARED_DATA_REF_KEY)
  1708. parent = ref->parent;
  1709. else
  1710. ref_root = ref->root;
  1711. if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
  1712. if (extent_op) {
  1713. BUG_ON(extent_op->update_key);
  1714. flags |= extent_op->flags_to_set;
  1715. }
  1716. ret = alloc_reserved_file_extent(trans, root,
  1717. parent, ref_root, flags,
  1718. ref->objectid, ref->offset,
  1719. &ins, node->ref_mod);
  1720. } else if (node->action == BTRFS_ADD_DELAYED_REF) {
  1721. ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
  1722. node->num_bytes, parent,
  1723. ref_root, ref->objectid,
  1724. ref->offset, node->ref_mod,
  1725. extent_op);
  1726. } else if (node->action == BTRFS_DROP_DELAYED_REF) {
  1727. ret = __btrfs_free_extent(trans, root, node->bytenr,
  1728. node->num_bytes, parent,
  1729. ref_root, ref->objectid,
  1730. ref->offset, node->ref_mod,
  1731. extent_op);
  1732. } else {
  1733. BUG();
  1734. }
  1735. return ret;
  1736. }
  1737. static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
  1738. struct extent_buffer *leaf,
  1739. struct btrfs_extent_item *ei)
  1740. {
  1741. u64 flags = btrfs_extent_flags(leaf, ei);
  1742. if (extent_op->update_flags) {
  1743. flags |= extent_op->flags_to_set;
  1744. btrfs_set_extent_flags(leaf, ei, flags);
  1745. }
  1746. if (extent_op->update_key) {
  1747. struct btrfs_tree_block_info *bi;
  1748. BUG_ON(!(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK));
  1749. bi = (struct btrfs_tree_block_info *)(ei + 1);
  1750. btrfs_set_tree_block_key(leaf, bi, &extent_op->key);
  1751. }
  1752. }
  1753. static int run_delayed_extent_op(struct btrfs_trans_handle *trans,
  1754. struct btrfs_root *root,
  1755. struct btrfs_delayed_ref_node *node,
  1756. struct btrfs_delayed_extent_op *extent_op)
  1757. {
  1758. struct btrfs_key key;
  1759. struct btrfs_path *path;
  1760. struct btrfs_extent_item *ei;
  1761. struct extent_buffer *leaf;
  1762. u32 item_size;
  1763. int ret;
  1764. int err = 0;
  1765. path = btrfs_alloc_path();
  1766. if (!path)
  1767. return -ENOMEM;
  1768. key.objectid = node->bytenr;
  1769. key.type = BTRFS_EXTENT_ITEM_KEY;
  1770. key.offset = node->num_bytes;
  1771. path->reada = 1;
  1772. path->leave_spinning = 1;
  1773. ret = btrfs_search_slot(trans, root->fs_info->extent_root, &key,
  1774. path, 0, 1);
  1775. if (ret < 0) {
  1776. err = ret;
  1777. goto out;
  1778. }
  1779. if (ret > 0) {
  1780. err = -EIO;
  1781. goto out;
  1782. }
  1783. leaf = path->nodes[0];
  1784. item_size = btrfs_item_size_nr(leaf, path->slots[0]);
  1785. #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
  1786. if (item_size < sizeof(*ei)) {
  1787. ret = convert_extent_item_v0(trans, root->fs_info->extent_root,
  1788. path, (u64)-1, 0);
  1789. if (ret < 0) {
  1790. err = ret;
  1791. goto out;
  1792. }
  1793. leaf = path->nodes[0];
  1794. item_size = btrfs_item_size_nr(leaf, path->slots[0]);
  1795. }
  1796. #endif
  1797. BUG_ON(item_size < sizeof(*ei));
  1798. ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
  1799. __run_delayed_extent_op(extent_op, leaf, ei);
  1800. btrfs_mark_buffer_dirty(leaf);
  1801. out:
  1802. btrfs_free_path(path);
  1803. return err;
  1804. }
  1805. static int run_delayed_tree_ref(struct btrfs_trans_handle *trans,
  1806. struct btrfs_root *root,
  1807. struct btrfs_delayed_ref_node *node,
  1808. struct btrfs_delayed_extent_op *extent_op,
  1809. int insert_reserved)
  1810. {
  1811. int ret = 0;
  1812. struct btrfs_delayed_tree_ref *ref;
  1813. struct btrfs_key ins;
  1814. u64 parent = 0;
  1815. u64 ref_root = 0;
  1816. ins.objectid = node->bytenr;
  1817. ins.offset = node->num_bytes;
  1818. ins.type = BTRFS_EXTENT_ITEM_KEY;
  1819. ref = btrfs_delayed_node_to_tree_ref(node);
  1820. if (node->type == BTRFS_SHARED_BLOCK_REF_KEY)
  1821. parent = ref->parent;
  1822. else
  1823. ref_root = ref->root;
  1824. BUG_ON(node->ref_mod != 1);
  1825. if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
  1826. BUG_ON(!extent_op || !extent_op->update_flags ||
  1827. !extent_op->update_key);
  1828. ret = alloc_reserved_tree_block(trans, root,
  1829. parent, ref_root,
  1830. extent_op->flags_to_set,
  1831. &extent_op->key,
  1832. ref->level, &ins);
  1833. } else if (node->action == BTRFS_ADD_DELAYED_REF) {
  1834. ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
  1835. node->num_bytes, parent, ref_root,
  1836. ref->level, 0, 1, extent_op);
  1837. } else if (node->action == BTRFS_DROP_DELAYED_REF) {
  1838. ret = __btrfs_free_extent(trans, root, node->bytenr,
  1839. node->num_bytes, parent, ref_root,
  1840. ref->level, 0, 1, extent_op);
  1841. } else {
  1842. BUG();
  1843. }
  1844. return ret;
  1845. }
  1846. /* helper function to actually process a single delayed ref entry */
  1847. static int run_one_delayed_ref(struct btrfs_trans_handle *trans,
  1848. struct btrfs_root *root,
  1849. struct btrfs_delayed_ref_node *node,
  1850. struct btrfs_delayed_extent_op *extent_op,
  1851. int insert_reserved)
  1852. {
  1853. int ret;
  1854. if (btrfs_delayed_ref_is_head(node)) {
  1855. struct btrfs_delayed_ref_head *head;
  1856. /*
  1857. * we've hit the end of the chain and we were supposed
  1858. * to insert this extent into the tree. But, it got
  1859. * deleted before we ever needed to insert it, so all
  1860. * we have to do is clean up the accounting
  1861. */
  1862. BUG_ON(extent_op);
  1863. head = btrfs_delayed_node_to_head(node);
  1864. if (insert_reserved) {
  1865. btrfs_pin_extent(root, node->bytenr,
  1866. node->num_bytes, 1);
  1867. if (head->is_data) {
  1868. ret = btrfs_del_csums(trans, root,
  1869. node->bytenr,
  1870. node->num_bytes);
  1871. BUG_ON(ret);
  1872. }
  1873. }
  1874. mutex_unlock(&head->mutex);
  1875. return 0;
  1876. }
  1877. if (node->type == BTRFS_TREE_BLOCK_REF_KEY ||
  1878. node->type == BTRFS_SHARED_BLOCK_REF_KEY)
  1879. ret = run_delayed_tree_ref(trans, root, node, extent_op,
  1880. insert_reserved);
  1881. else if (node->type == BTRFS_EXTENT_DATA_REF_KEY ||
  1882. node->type == BTRFS_SHARED_DATA_REF_KEY)
  1883. ret = run_delayed_data_ref(trans, root, node, extent_op,
  1884. insert_reserved);
  1885. else
  1886. BUG();
  1887. return ret;
  1888. }
  1889. static noinline struct btrfs_delayed_ref_node *
  1890. select_delayed_ref(struct btrfs_delayed_ref_head *head)
  1891. {
  1892. struct rb_node *node;
  1893. struct btrfs_delayed_ref_node *ref;
  1894. int action = BTRFS_ADD_DELAYED_REF;
  1895. again:
  1896. /*
  1897. * select delayed ref of type BTRFS_ADD_DELAYED_REF first.
  1898. * this prevents ref count from going down to zero when
  1899. * there still are pending delayed ref.
  1900. */
  1901. node = rb_prev(&head->node.rb_node);
  1902. while (1) {
  1903. if (!node)
  1904. break;
  1905. ref = rb_entry(node, struct btrfs_delayed_ref_node,
  1906. rb_node);
  1907. if (ref->bytenr != head->node.bytenr)
  1908. break;
  1909. if (ref->action == action)
  1910. return ref;
  1911. node = rb_prev(node);
  1912. }
  1913. if (action == BTRFS_ADD_DELAYED_REF) {
  1914. action = BTRFS_DROP_DELAYED_REF;
  1915. goto again;
  1916. }
  1917. return NULL;
  1918. }
  1919. static noinline int run_clustered_refs(struct btrfs_trans_handle *trans,
  1920. struct btrfs_root *root,
  1921. struct list_head *cluster)
  1922. {
  1923. struct btrfs_delayed_ref_root *delayed_refs;
  1924. struct btrfs_delayed_ref_node *ref;
  1925. struct btrfs_delayed_ref_head *locked_ref = NULL;
  1926. struct btrfs_delayed_extent_op *extent_op;
  1927. int ret;
  1928. int count = 0;
  1929. int must_insert_reserved = 0;
  1930. delayed_refs = &trans->transaction->delayed_refs;
  1931. while (1) {
  1932. if (!locked_ref) {
  1933. /* pick a new head ref from the cluster list */
  1934. if (list_empty(cluster))
  1935. break;
  1936. locked_ref = list_entry(cluster->next,
  1937. struct btrfs_delayed_ref_head, cluster);
  1938. /* grab the lock that says we are going to process
  1939. * all the refs for this head */
  1940. ret = btrfs_delayed_ref_lock(trans, locked_ref);
  1941. /*
  1942. * we may have dropped the spin lock to get the head
  1943. * mutex lock, and that might have given someone else
  1944. * time to free the head. If that's true, it has been
  1945. * removed from our list and we can move on.
  1946. */
  1947. if (ret == -EAGAIN) {
  1948. locked_ref = NULL;
  1949. count++;
  1950. continue;
  1951. }
  1952. }
  1953. /*
  1954. * record the must insert reserved flag before we
  1955. * drop the spin lock.
  1956. */
  1957. must_insert_reserved = locked_ref->must_insert_reserved;
  1958. locked_ref->must_insert_reserved = 0;
  1959. extent_op = locked_ref->extent_op;
  1960. locked_ref->extent_op = NULL;
  1961. /*
  1962. * locked_ref is the head node, so we have to go one
  1963. * node back for any delayed ref updates
  1964. */
  1965. ref = select_delayed_ref(locked_ref);
  1966. if (!ref) {
  1967. /* All delayed refs have been processed, Go ahead
  1968. * and send the head node to run_one_delayed_ref,
  1969. * so that any accounting fixes can happen
  1970. */
  1971. ref = &locked_ref->node;
  1972. if (extent_op && must_insert_reserved) {
  1973. kfree(extent_op);
  1974. extent_op = NULL;
  1975. }
  1976. if (extent_op) {
  1977. spin_unlock(&delayed_refs->lock);
  1978. ret = run_delayed_extent_op(trans, root,
  1979. ref, extent_op);
  1980. BUG_ON(ret);
  1981. kfree(extent_op);
  1982. cond_resched();
  1983. spin_lock(&delayed_refs->lock);
  1984. continue;
  1985. }
  1986. list_del_init(&locked_ref->cluster);
  1987. locked_ref = NULL;
  1988. }
  1989. ref->in_tree = 0;
  1990. rb_erase(&ref->rb_node, &delayed_refs->root);
  1991. delayed_refs->num_entries--;
  1992. spin_unlock(&delayed_refs->lock);
  1993. ret = run_one_delayed_ref(trans, root, ref, extent_op,
  1994. must_insert_reserved);
  1995. BUG_ON(ret);
  1996. btrfs_put_delayed_ref(ref);
  1997. kfree(extent_op);
  1998. count++;
  1999. cond_resched();
  2000. spin_lock(&delayed_refs->lock);
  2001. }
  2002. return count;
  2003. }
  2004. /*
  2005. * this starts processing the delayed reference count updates and
  2006. * extent insertions we have queued up so far. count can be
  2007. * 0, which means to process everything in the tree at the start
  2008. * of the run (but not newly added entries), or it can be some target
  2009. * number you'd like to process.
  2010. */
  2011. int btrfs_run_delayed_refs(struct btrfs_trans_handle *trans,
  2012. struct btrfs_root *root, unsigned long count)
  2013. {
  2014. struct rb_node *node;
  2015. struct btrfs_delayed_ref_root *delayed_refs;
  2016. struct btrfs_delayed_ref_node *ref;
  2017. struct list_head cluster;
  2018. int ret;
  2019. int run_all = count == (unsigned long)-1;
  2020. int run_most = 0;
  2021. if (root == root->fs_info->extent_root)
  2022. root = root->fs_info->tree_root;
  2023. delayed_refs = &trans->transaction->delayed_refs;
  2024. INIT_LIST_HEAD(&cluster);
  2025. again:
  2026. spin_lock(&delayed_refs->lock);
  2027. if (count == 0) {
  2028. count = delayed_refs->num_entries * 2;
  2029. run_most = 1;
  2030. }
  2031. while (1) {
  2032. if (!(run_all || run_most) &&
  2033. delayed_refs->num_heads_ready < 64)
  2034. break;
  2035. /*
  2036. * go find something we can process in the rbtree. We start at
  2037. * the beginning of the tree, and then build a cluster
  2038. * of refs to process starting at the first one we are able to
  2039. * lock
  2040. */
  2041. ret = btrfs_find_ref_cluster(trans, &cluster,
  2042. delayed_refs->run_delayed_start);
  2043. if (ret)
  2044. break;
  2045. ret = run_clustered_refs(trans, root, &cluster);
  2046. BUG_ON(ret < 0);
  2047. count -= min_t(unsigned long, ret, count);
  2048. if (count == 0)
  2049. break;
  2050. }
  2051. if (run_all) {
  2052. node = rb_first(&delayed_refs->root);
  2053. if (!node)
  2054. goto out;
  2055. count = (unsigned long)-1;
  2056. while (node) {
  2057. ref = rb_entry(node, struct btrfs_delayed_ref_node,
  2058. rb_node);
  2059. if (btrfs_delayed_ref_is_head(ref)) {
  2060. struct btrfs_delayed_ref_head *head;
  2061. head = btrfs_delayed_node_to_head(ref);
  2062. atomic_inc(&ref->refs);
  2063. spin_unlock(&delayed_refs->lock);
  2064. mutex_lock(&head->mutex);
  2065. mutex_unlock(&head->mutex);
  2066. btrfs_put_delayed_ref(ref);
  2067. cond_resched();
  2068. goto again;
  2069. }
  2070. node = rb_next(node);
  2071. }
  2072. spin_unlock(&delayed_refs->lock);
  2073. schedule_timeout(1);
  2074. goto again;
  2075. }
  2076. out:
  2077. spin_unlock(&delayed_refs->lock);
  2078. return 0;
  2079. }
  2080. int btrfs_set_disk_extent_flags(struct btrfs_trans_handle *trans,
  2081. struct btrfs_root *root,
  2082. u64 bytenr, u64 num_bytes, u64 flags,
  2083. int is_data)
  2084. {
  2085. struct btrfs_delayed_extent_op *extent_op;
  2086. int ret;
  2087. extent_op = kmalloc(sizeof(*extent_op), GFP_NOFS);
  2088. if (!extent_op)
  2089. return -ENOMEM;
  2090. extent_op->flags_to_set = flags;
  2091. extent_op->update_flags = 1;
  2092. extent_op->update_key = 0;
  2093. extent_op->is_data = is_data ? 1 : 0;
  2094. ret = btrfs_add_delayed_extent_op(trans, bytenr, num_bytes, extent_op);
  2095. if (ret)
  2096. kfree(extent_op);
  2097. return ret;
  2098. }
  2099. static noinline int check_delayed_ref(struct btrfs_trans_handle *trans,
  2100. struct btrfs_root *root,
  2101. struct btrfs_path *path,
  2102. u64 objectid, u64 offset, u64 bytenr)
  2103. {
  2104. struct btrfs_delayed_ref_head *head;
  2105. struct btrfs_delayed_ref_node *ref;
  2106. struct btrfs_delayed_data_ref *data_ref;
  2107. struct btrfs_delayed_ref_root *delayed_refs;
  2108. struct rb_node *node;
  2109. int ret = 0;
  2110. ret = -ENOENT;
  2111. delayed_refs = &trans->transaction->delayed_refs;
  2112. spin_lock(&delayed_refs->lock);
  2113. head = btrfs_find_delayed_ref_head(trans, bytenr);
  2114. if (!head)
  2115. goto out;
  2116. if (!mutex_trylock(&head->mutex)) {
  2117. atomic_inc(&head->node.refs);
  2118. spin_unlock(&delayed_refs->lock);
  2119. btrfs_release_path(root->fs_info->extent_root, path);
  2120. mutex_lock(&head->mutex);
  2121. mutex_unlock(&head->mutex);
  2122. btrfs_put_delayed_ref(&head->node);
  2123. return -EAGAIN;
  2124. }
  2125. node = rb_prev(&head->node.rb_node);
  2126. if (!node)
  2127. goto out_unlock;
  2128. ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);
  2129. if (ref->bytenr != bytenr)
  2130. goto out_unlock;
  2131. ret = 1;
  2132. if (ref->type != BTRFS_EXTENT_DATA_REF_KEY)
  2133. goto out_unlock;
  2134. data_ref = btrfs_delayed_node_to_data_ref(ref);
  2135. node = rb_prev(node);
  2136. if (node) {
  2137. ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);
  2138. if (ref->bytenr == bytenr)
  2139. goto out_unlock;
  2140. }
  2141. if (data_ref->root != root->root_key.objectid ||
  2142. data_ref->objectid != objectid || data_ref->offset != offset)
  2143. goto out_unlock;
  2144. ret = 0;
  2145. out_unlock:
  2146. mutex_unlock(&head->mutex);
  2147. out:
  2148. spin_unlock(&delayed_refs->lock);
  2149. return ret;
  2150. }
  2151. static noinline int check_committed_ref(struct btrfs_trans_handle *trans,
  2152. struct btrfs_root *root,
  2153. struct btrfs_path *path,
  2154. u64 objectid, u64 offset, u64 bytenr)
  2155. {
  2156. struct btrfs_root *extent_root = root->fs_info->extent_root;
  2157. struct extent_buffer *leaf;
  2158. struct btrfs_extent_data_ref *ref;
  2159. struct btrfs_extent_inline_ref *iref;
  2160. struct btrfs_extent_item *ei;
  2161. struct btrfs_key key;
  2162. u32 item_size;
  2163. int ret;
  2164. key.objectid = bytenr;
  2165. key.offset = (u64)-1;
  2166. key.type = BTRFS_EXTENT_ITEM_KEY;
  2167. ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
  2168. if (ret < 0)
  2169. goto out;
  2170. BUG_ON(ret == 0);
  2171. ret = -ENOENT;
  2172. if (path->slots[0] == 0)
  2173. goto out;
  2174. path->slots[0]--;
  2175. leaf = path->nodes[0];
  2176. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  2177. if (key.objectid != bytenr || key.type != BTRFS_EXTENT_ITEM_KEY)
  2178. goto out;
  2179. ret = 1;
  2180. item_size = btrfs_item_size_nr(leaf, path->slots[0]);
  2181. #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
  2182. if (item_size < sizeof(*ei)) {
  2183. WARN_ON(item_size != sizeof(struct btrfs_extent_item_v0));
  2184. goto out;
  2185. }
  2186. #endif
  2187. ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
  2188. if (item_size != sizeof(*ei) +
  2189. btrfs_extent_inline_ref_size(BTRFS_EXTENT_DATA_REF_KEY))
  2190. goto out;
  2191. if (btrfs_extent_generation(leaf, ei) <=
  2192. btrfs_root_last_snapshot(&root->root_item))
  2193. goto out;
  2194. iref = (struct btrfs_extent_inline_ref *)(ei + 1);
  2195. if (btrfs_extent_inline_ref_type(leaf, iref) !=
  2196. BTRFS_EXTENT_DATA_REF_KEY)
  2197. goto out;
  2198. ref = (struct btrfs_extent_data_ref *)(&iref->offset);
  2199. if (btrfs_extent_refs(leaf, ei) !=
  2200. btrfs_extent_data_ref_count(leaf, ref) ||
  2201. btrfs_extent_data_ref_root(leaf, ref) !=
  2202. root->root_key.objectid ||
  2203. btrfs_extent_data_ref_objectid(leaf, ref) != objectid ||
  2204. btrfs_extent_data_ref_offset(leaf, ref) != offset)
  2205. goto out;
  2206. ret = 0;
  2207. out:
  2208. return ret;
  2209. }
  2210. int btrfs_cross_ref_exist(struct btrfs_trans_handle *trans,
  2211. struct btrfs_root *root,
  2212. u64 objectid, u64 offset, u64 bytenr)
  2213. {
  2214. struct btrfs_path *path;
  2215. int ret;
  2216. int ret2;
  2217. path = btrfs_alloc_path();
  2218. if (!path)
  2219. return -ENOENT;
  2220. do {
  2221. ret = check_committed_ref(trans, root, path, objectid,
  2222. offset, bytenr);
  2223. if (ret && ret != -ENOENT)
  2224. goto out;
  2225. ret2 = check_delayed_ref(trans, root, path, objectid,
  2226. offset, bytenr);
  2227. } while (ret2 == -EAGAIN);
  2228. if (ret2 && ret2 != -ENOENT) {
  2229. ret = ret2;
  2230. goto out;
  2231. }
  2232. if (ret != -ENOENT || ret2 != -ENOENT)
  2233. ret = 0;
  2234. out:
  2235. btrfs_free_path(path);
  2236. if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
  2237. WARN_ON(ret > 0);
  2238. return ret;
  2239. }
  2240. #if 0
  2241. int btrfs_cache_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
  2242. struct extent_buffer *buf, u32 nr_extents)
  2243. {
  2244. struct btrfs_key key;
  2245. struct btrfs_file_extent_item *fi;
  2246. u64 root_gen;
  2247. u32 nritems;
  2248. int i;
  2249. int level;
  2250. int ret = 0;
  2251. int shared = 0;
  2252. if (!root->ref_cows)
  2253. return 0;
  2254. if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) {
  2255. shared = 0;
  2256. root_gen = root->root_key.offset;
  2257. } else {
  2258. shared = 1;
  2259. root_gen = trans->transid - 1;
  2260. }
  2261. level = btrfs_header_level(buf);
  2262. nritems = btrfs_header_nritems(buf);
  2263. if (level == 0) {
  2264. struct btrfs_leaf_ref *ref;
  2265. struct btrfs_extent_info *info;
  2266. ref = btrfs_alloc_leaf_ref(root, nr_extents);
  2267. if (!ref) {
  2268. ret = -ENOMEM;
  2269. goto out;
  2270. }
  2271. ref->root_gen = root_gen;
  2272. ref->bytenr = buf->start;
  2273. ref->owner = btrfs_header_owner(buf);
  2274. ref->generation = btrfs_header_generation(buf);
  2275. ref->nritems = nr_extents;
  2276. info = ref->extents;
  2277. for (i = 0; nr_extents > 0 && i < nritems; i++) {
  2278. u64 disk_bytenr;
  2279. btrfs_item_key_to_cpu(buf, &key, i);
  2280. if (btrfs_key_type(&key) != BTRFS_EXTENT_DATA_KEY)
  2281. continue;
  2282. fi = btrfs_item_ptr(buf, i,
  2283. struct btrfs_file_extent_item);
  2284. if (btrfs_file_extent_type(buf, fi) ==
  2285. BTRFS_FILE_EXTENT_INLINE)
  2286. continue;
  2287. disk_bytenr = btrfs_file_extent_disk_bytenr(buf, fi);
  2288. if (disk_bytenr == 0)
  2289. continue;
  2290. info->bytenr = disk_bytenr;
  2291. info->num_bytes =
  2292. btrfs_file_extent_disk_num_bytes(buf, fi);
  2293. info->objectid = key.objectid;
  2294. info->offset = key.offset;
  2295. info++;
  2296. }
  2297. ret = btrfs_add_leaf_ref(root, ref, shared);
  2298. if (ret == -EEXIST && shared) {
  2299. struct btrfs_leaf_ref *old;
  2300. old = btrfs_lookup_leaf_ref(root, ref->bytenr);
  2301. BUG_ON(!old);
  2302. btrfs_remove_leaf_ref(root, old);
  2303. btrfs_free_leaf_ref(root, old);
  2304. ret = btrfs_add_leaf_ref(root, ref, shared);
  2305. }
  2306. WARN_ON(ret);
  2307. btrfs_free_leaf_ref(root, ref);
  2308. }
  2309. out:
  2310. return ret;
  2311. }
  2312. /* when a block goes through cow, we update the reference counts of
  2313. * everything that block points to. The internal pointers of the block
  2314. * can be in just about any order, and it is likely to have clusters of
  2315. * things that are close together and clusters of things that are not.
  2316. *
  2317. * To help reduce the seeks that come with updating all of these reference
  2318. * counts, sort them by byte number before actual updates are done.
  2319. *
  2320. * struct refsort is used to match byte number to slot in the btree block.
  2321. * we sort based on the byte number and then use the slot to actually
  2322. * find the item.
  2323. *
  2324. * struct refsort is smaller than strcut btrfs_item and smaller than
  2325. * struct btrfs_key_ptr. Since we're currently limited to the page size
  2326. * for a btree block, there's no way for a kmalloc of refsorts for a
  2327. * single node to be bigger than a page.
  2328. */
  2329. struct refsort {
  2330. u64 bytenr;
  2331. u32 slot;
  2332. };
  2333. /*
  2334. * for passing into sort()
  2335. */
  2336. static int refsort_cmp(const void *a_void, const void *b_void)
  2337. {
  2338. const struct refsort *a = a_void;
  2339. const struct refsort *b = b_void;
  2340. if (a->bytenr < b->bytenr)
  2341. return -1;
  2342. if (a->bytenr > b->bytenr)
  2343. return 1;
  2344. return 0;
  2345. }
  2346. #endif
  2347. static int __btrfs_mod_ref(struct btrfs_trans_handle *trans,
  2348. struct btrfs_root *root,
  2349. struct extent_buffer *buf,
  2350. int full_backref, int inc)
  2351. {
  2352. u64 bytenr;
  2353. u64 num_bytes;
  2354. u64 parent;
  2355. u64 ref_root;
  2356. u32 nritems;
  2357. struct btrfs_key key;
  2358. struct btrfs_file_extent_item *fi;
  2359. int i;
  2360. int level;
  2361. int ret = 0;
  2362. int (*process_func)(struct btrfs_trans_handle *, struct btrfs_root *,
  2363. u64, u64, u64, u64, u64, u64);
  2364. ref_root = btrfs_header_owner(buf);
  2365. nritems = btrfs_header_nritems(buf);
  2366. level = btrfs_header_level(buf);
  2367. if (!root->ref_cows && level == 0)
  2368. return 0;
  2369. if (inc)
  2370. process_func = btrfs_inc_extent_ref;
  2371. else
  2372. process_func = btrfs_free_extent;
  2373. if (full_backref)
  2374. parent = buf->start;
  2375. else
  2376. parent = 0;
  2377. for (i = 0; i < nritems; i++) {
  2378. if (level == 0) {
  2379. btrfs_item_key_to_cpu(buf, &key, i);
  2380. if (btrfs_key_type(&key) != BTRFS_EXTENT_DATA_KEY)
  2381. continue;
  2382. fi = btrfs_item_ptr(buf, i,
  2383. struct btrfs_file_extent_item);
  2384. if (btrfs_file_extent_type(buf, fi) ==
  2385. BTRFS_FILE_EXTENT_INLINE)
  2386. continue;
  2387. bytenr = btrfs_file_extent_disk_bytenr(buf, fi);
  2388. if (bytenr == 0)
  2389. continue;
  2390. num_bytes = btrfs_file_extent_disk_num_bytes(buf, fi);
  2391. key.offset -= btrfs_file_extent_offset(buf, fi);
  2392. ret = process_func(trans, root, bytenr, num_bytes,
  2393. parent, ref_root, key.objectid,
  2394. key.offset);
  2395. if (ret)
  2396. goto fail;
  2397. } else {
  2398. bytenr = btrfs_node_blockptr(buf, i);
  2399. num_bytes = btrfs_level_size(root, level - 1);
  2400. ret = process_func(trans, root, bytenr, num_bytes,
  2401. parent, ref_root, level - 1, 0);
  2402. if (ret)
  2403. goto fail;
  2404. }
  2405. }
  2406. return 0;
  2407. fail:
  2408. BUG();
  2409. return ret;
  2410. }
  2411. int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
  2412. struct extent_buffer *buf, int full_backref)
  2413. {
  2414. return __btrfs_mod_ref(trans, root, buf, full_backref, 1);
  2415. }
  2416. int btrfs_dec_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
  2417. struct extent_buffer *buf, int full_backref)
  2418. {
  2419. return __btrfs_mod_ref(trans, root, buf, full_backref, 0);
  2420. }
  2421. static int write_one_cache_group(struct btrfs_trans_handle *trans,
  2422. struct btrfs_root *root,
  2423. struct btrfs_path *path,
  2424. struct btrfs_block_group_cache *cache)
  2425. {
  2426. int ret;
  2427. struct btrfs_root *extent_root = root->fs_info->extent_root;
  2428. unsigned long bi;
  2429. struct extent_buffer *leaf;
  2430. ret = btrfs_search_slot(trans, extent_root, &cache->key, path, 0, 1);
  2431. if (ret < 0)
  2432. goto fail;
  2433. BUG_ON(ret);
  2434. leaf = path->nodes[0];
  2435. bi = btrfs_item_ptr_offset(leaf, path->slots[0]);
  2436. write_extent_buffer(leaf, &cache->item, bi, sizeof(cache->item));
  2437. btrfs_mark_buffer_dirty(leaf);
  2438. btrfs_release_path(extent_root, path);
  2439. fail:
  2440. if (ret)
  2441. return ret;
  2442. return 0;
  2443. }
  2444. static struct btrfs_block_group_cache *
  2445. next_block_group(struct btrfs_root *root,
  2446. struct btrfs_block_group_cache *cache)
  2447. {
  2448. struct rb_node *node;
  2449. spin_lock(&root->fs_info->block_group_cache_lock);
  2450. node = rb_next(&cache->cache_node);
  2451. btrfs_put_block_group(cache);
  2452. if (node) {
  2453. cache = rb_entry(node, struct btrfs_block_group_cache,
  2454. cache_node);
  2455. btrfs_get_block_group(cache);
  2456. } else
  2457. cache = NULL;
  2458. spin_unlock(&root->fs_info->block_group_cache_lock);
  2459. return cache;
  2460. }
  2461. static int cache_save_setup(struct btrfs_block_group_cache *block_group,
  2462. struct btrfs_trans_handle *trans,
  2463. struct btrfs_path *path)
  2464. {
  2465. struct btrfs_root *root = block_group->fs_info->tree_root;
  2466. struct inode *inode = NULL;
  2467. u64 alloc_hint = 0;
  2468. int dcs = BTRFS_DC_ERROR;
  2469. int num_pages = 0;
  2470. int retries = 0;
  2471. int ret = 0;
  2472. /*
  2473. * If this block group is smaller than 100 megs don't bother caching the
  2474. * block group.
  2475. */
  2476. if (block_group->key.offset < (100 * 1024 * 1024)) {
  2477. spin_lock(&block_group->lock);
  2478. block_group->disk_cache_state = BTRFS_DC_WRITTEN;
  2479. spin_unlock(&block_group->lock);
  2480. return 0;
  2481. }
  2482. again:
  2483. inode = lookup_free_space_inode(root, block_group, path);
  2484. if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) {
  2485. ret = PTR_ERR(inode);
  2486. btrfs_release_path(root, path);
  2487. goto out;
  2488. }
  2489. if (IS_ERR(inode)) {
  2490. BUG_ON(retries);
  2491. retries++;
  2492. if (block_group->ro)
  2493. goto out_free;
  2494. ret = create_free_space_inode(root, trans, block_group, path);
  2495. if (ret)
  2496. goto out_free;
  2497. goto again;
  2498. }
  2499. /*
  2500. * We want to set the generation to 0, that way if anything goes wrong
  2501. * from here on out we know not to trust this cache when we load up next
  2502. * time.
  2503. */
  2504. BTRFS_I(inode)->generation = 0;
  2505. ret = btrfs_update_inode(trans, root, inode);
  2506. WARN_ON(ret);
  2507. if (i_size_read(inode) > 0) {
  2508. ret = btrfs_truncate_free_space_cache(root, trans, path,
  2509. inode);
  2510. if (ret)
  2511. goto out_put;
  2512. }
  2513. spin_lock(&block_group->lock);
  2514. if (block_group->cached != BTRFS_CACHE_FINISHED) {
  2515. /* We're not cached, don't bother trying to write stuff out */
  2516. dcs = BTRFS_DC_WRITTEN;
  2517. spin_unlock(&block_group->lock);
  2518. goto out_put;
  2519. }
  2520. spin_unlock(&block_group->lock);
  2521. num_pages = (int)div64_u64(block_group->key.offset, 1024 * 1024 * 1024);
  2522. if (!num_pages)
  2523. num_pages = 1;
  2524. /*
  2525. * Just to make absolutely sure we have enough space, we're going to
  2526. * preallocate 12 pages worth of space for each block group. In
  2527. * practice we ought to use at most 8, but we need extra space so we can
  2528. * add our header and have a terminator between the extents and the
  2529. * bitmaps.
  2530. */
  2531. num_pages *= 16;
  2532. num_pages *= PAGE_CACHE_SIZE;
  2533. ret = btrfs_check_data_free_space(inode, num_pages);
  2534. if (ret)
  2535. goto out_put;
  2536. ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, num_pages,
  2537. num_pages, num_pages,
  2538. &alloc_hint);
  2539. if (!ret)
  2540. dcs = BTRFS_DC_SETUP;
  2541. btrfs_free_reserved_data_space(inode, num_pages);
  2542. out_put:
  2543. iput(inode);
  2544. out_free:
  2545. btrfs_release_path(root, path);
  2546. out:
  2547. spin_lock(&block_group->lock);
  2548. block_group->disk_cache_state = dcs;
  2549. spin_unlock(&block_group->lock);
  2550. return ret;
  2551. }
  2552. int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans,
  2553. struct btrfs_root *root)
  2554. {
  2555. struct btrfs_block_group_cache *cache;
  2556. int err = 0;
  2557. struct btrfs_path *path;
  2558. u64 last = 0;
  2559. path = btrfs_alloc_path();
  2560. if (!path)
  2561. return -ENOMEM;
  2562. again:
  2563. while (1) {
  2564. cache = btrfs_lookup_first_block_group(root->fs_info, last);
  2565. while (cache) {
  2566. if (cache->disk_cache_state == BTRFS_DC_CLEAR)
  2567. break;
  2568. cache = next_block_group(root, cache);
  2569. }
  2570. if (!cache) {
  2571. if (last == 0)
  2572. break;
  2573. last = 0;
  2574. continue;
  2575. }
  2576. err = cache_save_setup(cache, trans, path);
  2577. last = cache->key.objectid + cache->key.offset;
  2578. btrfs_put_block_group(cache);
  2579. }
  2580. while (1) {
  2581. if (last == 0) {
  2582. err = btrfs_run_delayed_refs(trans, root,
  2583. (unsigned long)-1);
  2584. BUG_ON(err);
  2585. }
  2586. cache = btrfs_lookup_first_block_group(root->fs_info, last);
  2587. while (cache) {
  2588. if (cache->disk_cache_state == BTRFS_DC_CLEAR) {
  2589. btrfs_put_block_group(cache);
  2590. goto again;
  2591. }
  2592. if (cache->dirty)
  2593. break;
  2594. cache = next_block_group(root, cache);
  2595. }
  2596. if (!cache) {
  2597. if (last == 0)
  2598. break;
  2599. last = 0;
  2600. continue;
  2601. }
  2602. if (cache->disk_cache_state == BTRFS_DC_SETUP)
  2603. cache->disk_cache_state = BTRFS_DC_NEED_WRITE;
  2604. cache->dirty = 0;
  2605. last = cache->key.objectid + cache->key.offset;
  2606. err = write_one_cache_group(trans, root, path, cache);
  2607. BUG_ON(err);
  2608. btrfs_put_block_group(cache);
  2609. }
  2610. while (1) {
  2611. /*
  2612. * I don't think this is needed since we're just marking our
  2613. * preallocated extent as written, but just in case it can't
  2614. * hurt.
  2615. */
  2616. if (last == 0) {
  2617. err = btrfs_run_delayed_refs(trans, root,
  2618. (unsigned long)-1);
  2619. BUG_ON(err);
  2620. }
  2621. cache = btrfs_lookup_first_block_group(root->fs_info, last);
  2622. while (cache) {
  2623. /*
  2624. * Really this shouldn't happen, but it could if we
  2625. * couldn't write the entire preallocated extent and
  2626. * splitting the extent resulted in a new block.
  2627. */
  2628. if (cache->dirty) {
  2629. btrfs_put_block_group(cache);
  2630. goto again;
  2631. }
  2632. if (cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
  2633. break;
  2634. cache = next_block_group(root, cache);
  2635. }
  2636. if (!cache) {
  2637. if (last == 0)
  2638. break;
  2639. last = 0;
  2640. continue;
  2641. }
  2642. btrfs_write_out_cache(root, trans, cache, path);
  2643. /*
  2644. * If we didn't have an error then the cache state is still
  2645. * NEED_WRITE, so we can set it to WRITTEN.
  2646. */
  2647. if (cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
  2648. cache->disk_cache_state = BTRFS_DC_WRITTEN;
  2649. last = cache->key.objectid + cache->key.offset;
  2650. btrfs_put_block_group(cache);
  2651. }
  2652. btrfs_free_path(path);
  2653. return 0;
  2654. }
  2655. int btrfs_extent_readonly(struct btrfs_root *root, u64 bytenr)
  2656. {
  2657. struct btrfs_block_group_cache *block_group;
  2658. int readonly = 0;
  2659. block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
  2660. if (!block_group || block_group->ro)
  2661. readonly = 1;
  2662. if (block_group)
  2663. btrfs_put_block_group(block_group);
  2664. return readonly;
  2665. }
  2666. static int update_space_info(struct btrfs_fs_info *info, u64 flags,
  2667. u64 total_bytes, u64 bytes_used,
  2668. struct btrfs_space_info **space_info)
  2669. {
  2670. struct btrfs_space_info *found;
  2671. int i;
  2672. int factor;
  2673. if (flags & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1 |
  2674. BTRFS_BLOCK_GROUP_RAID10))
  2675. factor = 2;
  2676. else
  2677. factor = 1;
  2678. found = __find_space_info(info, flags);
  2679. if (found) {
  2680. spin_lock(&found->lock);
  2681. found->total_bytes += total_bytes;
  2682. found->disk_total += total_bytes * factor;
  2683. found->bytes_used += bytes_used;
  2684. found->disk_used += bytes_used * factor;
  2685. found->full = 0;
  2686. spin_unlock(&found->lock);
  2687. *space_info = found;
  2688. return 0;
  2689. }
  2690. found = kzalloc(sizeof(*found), GFP_NOFS);
  2691. if (!found)
  2692. return -ENOMEM;
  2693. for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
  2694. INIT_LIST_HEAD(&found->block_groups[i]);
  2695. init_rwsem(&found->groups_sem);
  2696. spin_lock_init(&found->lock);
  2697. found->flags = flags & (BTRFS_BLOCK_GROUP_DATA |
  2698. BTRFS_BLOCK_GROUP_SYSTEM |
  2699. BTRFS_BLOCK_GROUP_METADATA);
  2700. found->total_bytes = total_bytes;
  2701. found->disk_total = total_bytes * factor;
  2702. found->bytes_used = bytes_used;
  2703. found->disk_used = bytes_used * factor;
  2704. found->bytes_pinned = 0;
  2705. found->bytes_reserved = 0;
  2706. found->bytes_readonly = 0;
  2707. found->bytes_may_use = 0;
  2708. found->full = 0;
  2709. found->force_alloc = CHUNK_ALLOC_NO_FORCE;
  2710. found->chunk_alloc = 0;
  2711. *space_info = found;
  2712. list_add_rcu(&found->list, &info->space_info);
  2713. atomic_set(&found->caching_threads, 0);
  2714. return 0;
  2715. }
  2716. static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
  2717. {
  2718. u64 extra_flags = flags & (BTRFS_BLOCK_GROUP_RAID0 |
  2719. BTRFS_BLOCK_GROUP_RAID1 |
  2720. BTRFS_BLOCK_GROUP_RAID10 |
  2721. BTRFS_BLOCK_GROUP_DUP);
  2722. if (extra_flags) {
  2723. if (flags & BTRFS_BLOCK_GROUP_DATA)
  2724. fs_info->avail_data_alloc_bits |= extra_flags;
  2725. if (flags & BTRFS_BLOCK_GROUP_METADATA)
  2726. fs_info->avail_metadata_alloc_bits |= extra_flags;
  2727. if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
  2728. fs_info->avail_system_alloc_bits |= extra_flags;
  2729. }
  2730. }
  2731. u64 btrfs_reduce_alloc_profile(struct btrfs_root *root, u64 flags)
  2732. {
  2733. /*
  2734. * we add in the count of missing devices because we want
  2735. * to make sure that any RAID levels on a degraded FS
  2736. * continue to be honored.
  2737. */
  2738. u64 num_devices = root->fs_info->fs_devices->rw_devices +
  2739. root->fs_info->fs_devices->missing_devices;
  2740. if (num_devices == 1)
  2741. flags &= ~(BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID0);
  2742. if (num_devices < 4)
  2743. flags &= ~BTRFS_BLOCK_GROUP_RAID10;
  2744. if ((flags & BTRFS_BLOCK_GROUP_DUP) &&
  2745. (flags & (BTRFS_BLOCK_GROUP_RAID1 |
  2746. BTRFS_BLOCK_GROUP_RAID10))) {
  2747. flags &= ~BTRFS_BLOCK_GROUP_DUP;
  2748. }
  2749. if ((flags & BTRFS_BLOCK_GROUP_RAID1) &&
  2750. (flags & BTRFS_BLOCK_GROUP_RAID10)) {
  2751. flags &= ~BTRFS_BLOCK_GROUP_RAID1;
  2752. }
  2753. if ((flags & BTRFS_BLOCK_GROUP_RAID0) &&
  2754. ((flags & BTRFS_BLOCK_GROUP_RAID1) |
  2755. (flags & BTRFS_BLOCK_GROUP_RAID10) |
  2756. (flags & BTRFS_BLOCK_GROUP_DUP)))
  2757. flags &= ~BTRFS_BLOCK_GROUP_RAID0;
  2758. return flags;
  2759. }
  2760. static u64 get_alloc_profile(struct btrfs_root *root, u64 flags)
  2761. {
  2762. if (flags & BTRFS_BLOCK_GROUP_DATA)
  2763. flags |= root->fs_info->avail_data_alloc_bits &
  2764. root->fs_info->data_alloc_profile;
  2765. else if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
  2766. flags |= root->fs_info->avail_system_alloc_bits &
  2767. root->fs_info->system_alloc_profile;
  2768. else if (flags & BTRFS_BLOCK_GROUP_METADATA)
  2769. flags |= root->fs_info->avail_metadata_alloc_bits &
  2770. root->fs_info->metadata_alloc_profile;
  2771. return btrfs_reduce_alloc_profile(root, flags);
  2772. }
  2773. u64 btrfs_get_alloc_profile(struct btrfs_root *root, int data)
  2774. {
  2775. u64 flags;
  2776. if (data)
  2777. flags = BTRFS_BLOCK_GROUP_DATA;
  2778. else if (root == root->fs_info->chunk_root)
  2779. flags = BTRFS_BLOCK_GROUP_SYSTEM;
  2780. else
  2781. flags = BTRFS_BLOCK_GROUP_METADATA;
  2782. return get_alloc_profile(root, flags);
  2783. }
  2784. void btrfs_set_inode_space_info(struct btrfs_root *root, struct inode *inode)
  2785. {
  2786. BTRFS_I(inode)->space_info = __find_space_info(root->fs_info,
  2787. BTRFS_BLOCK_GROUP_DATA);
  2788. }
  2789. /*
  2790. * This will check the space that the inode allocates from to make sure we have
  2791. * enough space for bytes.
  2792. */
  2793. int btrfs_check_data_free_space(struct inode *inode, u64 bytes)
  2794. {
  2795. struct btrfs_space_info *data_sinfo;
  2796. struct btrfs_root *root = BTRFS_I(inode)->root;
  2797. u64 used;
  2798. int ret = 0, committed = 0, alloc_chunk = 1;
  2799. /* make sure bytes are sectorsize aligned */
  2800. bytes = (bytes + root->sectorsize - 1) & ~((u64)root->sectorsize - 1);
  2801. if (root == root->fs_info->tree_root ||
  2802. BTRFS_I(inode)->location.objectid == BTRFS_FREE_INO_OBJECTID) {
  2803. alloc_chunk = 0;
  2804. committed = 1;
  2805. }
  2806. data_sinfo = BTRFS_I(inode)->space_info;
  2807. if (!data_sinfo)
  2808. goto alloc;
  2809. again:
  2810. /* make sure we have enough space to handle the data first */
  2811. spin_lock(&data_sinfo->lock);
  2812. used = data_sinfo->bytes_used + data_sinfo->bytes_reserved +
  2813. data_sinfo->bytes_pinned + data_sinfo->bytes_readonly +
  2814. data_sinfo->bytes_may_use;
  2815. if (used + bytes > data_sinfo->total_bytes) {
  2816. struct btrfs_trans_handle *trans;
  2817. /*
  2818. * if we don't have enough free bytes in this space then we need
  2819. * to alloc a new chunk.
  2820. */
  2821. if (!data_sinfo->full && alloc_chunk) {
  2822. u64 alloc_target;
  2823. data_sinfo->force_alloc = CHUNK_ALLOC_FORCE;
  2824. spin_unlock(&data_sinfo->lock);
  2825. alloc:
  2826. alloc_target = btrfs_get_alloc_profile(root, 1);
  2827. trans = btrfs_join_transaction(root, 1);
  2828. if (IS_ERR(trans))
  2829. return PTR_ERR(trans);
  2830. ret = do_chunk_alloc(trans, root->fs_info->extent_root,
  2831. bytes + 2 * 1024 * 1024,
  2832. alloc_target,
  2833. CHUNK_ALLOC_NO_FORCE);
  2834. btrfs_end_transaction(trans, root);
  2835. if (ret < 0) {
  2836. if (ret != -ENOSPC)
  2837. return ret;
  2838. else
  2839. goto commit_trans;
  2840. }
  2841. if (!data_sinfo) {
  2842. btrfs_set_inode_space_info(root, inode);
  2843. data_sinfo = BTRFS_I(inode)->space_info;
  2844. }
  2845. goto again;
  2846. }
  2847. spin_unlock(&data_sinfo->lock);
  2848. /* commit the current transaction and try again */
  2849. commit_trans:
  2850. if (!committed && !root->fs_info->open_ioctl_trans) {
  2851. committed = 1;
  2852. trans = btrfs_join_transaction(root, 1);
  2853. if (IS_ERR(trans))
  2854. return PTR_ERR(trans);
  2855. ret = btrfs_commit_transaction(trans, root);
  2856. if (ret)
  2857. return ret;
  2858. goto again;
  2859. }
  2860. #if 0 /* I hope we never need this code again, just in case */
  2861. printk(KERN_ERR "no space left, need %llu, %llu bytes_used, "
  2862. "%llu bytes_reserved, " "%llu bytes_pinned, "
  2863. "%llu bytes_readonly, %llu may use %llu total\n",
  2864. (unsigned long long)bytes,
  2865. (unsigned long long)data_sinfo->bytes_used,
  2866. (unsigned long long)data_sinfo->bytes_reserved,
  2867. (unsigned long long)data_sinfo->bytes_pinned,
  2868. (unsigned long long)data_sinfo->bytes_readonly,
  2869. (unsigned long long)data_sinfo->bytes_may_use,
  2870. (unsigned long long)data_sinfo->total_bytes);
  2871. #endif
  2872. return -ENOSPC;
  2873. }
  2874. data_sinfo->bytes_may_use += bytes;
  2875. BTRFS_I(inode)->reserved_bytes += bytes;
  2876. spin_unlock(&data_sinfo->lock);
  2877. return 0;
  2878. }
  2879. /*
  2880. * called when we are clearing an delalloc extent from the
  2881. * inode's io_tree or there was an error for whatever reason
  2882. * after calling btrfs_check_data_free_space
  2883. */
  2884. void btrfs_free_reserved_data_space(struct inode *inode, u64 bytes)
  2885. {
  2886. struct btrfs_root *root = BTRFS_I(inode)->root;
  2887. struct btrfs_space_info *data_sinfo;
  2888. /* make sure bytes are sectorsize aligned */
  2889. bytes = (bytes + root->sectorsize - 1) & ~((u64)root->sectorsize - 1);
  2890. data_sinfo = BTRFS_I(inode)->space_info;
  2891. spin_lock(&data_sinfo->lock);
  2892. data_sinfo->bytes_may_use -= bytes;
  2893. BTRFS_I(inode)->reserved_bytes -= bytes;
  2894. spin_unlock(&data_sinfo->lock);
  2895. }
  2896. static void force_metadata_allocation(struct btrfs_fs_info *info)
  2897. {
  2898. struct list_head *head = &info->space_info;
  2899. struct btrfs_space_info *found;
  2900. rcu_read_lock();
  2901. list_for_each_entry_rcu(found, head, list) {
  2902. if (found->flags & BTRFS_BLOCK_GROUP_METADATA)
  2903. found->force_alloc = CHUNK_ALLOC_FORCE;
  2904. }
  2905. rcu_read_unlock();
  2906. }
  2907. static int should_alloc_chunk(struct btrfs_root *root,
  2908. struct btrfs_space_info *sinfo, u64 alloc_bytes,
  2909. int force)
  2910. {
  2911. u64 num_bytes = sinfo->total_bytes - sinfo->bytes_readonly;
  2912. u64 num_allocated = sinfo->bytes_used + sinfo->bytes_reserved;
  2913. u64 thresh;
  2914. if (force == CHUNK_ALLOC_FORCE)
  2915. return 1;
  2916. /*
  2917. * in limited mode, we want to have some free space up to
  2918. * about 1% of the FS size.
  2919. */
  2920. if (force == CHUNK_ALLOC_LIMITED) {
  2921. thresh = btrfs_super_total_bytes(&root->fs_info->super_copy);
  2922. thresh = max_t(u64, 64 * 1024 * 1024,
  2923. div_factor_fine(thresh, 1));
  2924. if (num_bytes - num_allocated < thresh)
  2925. return 1;
  2926. }
  2927. /*
  2928. * we have two similar checks here, one based on percentage
  2929. * and once based on a hard number of 256MB. The idea
  2930. * is that if we have a good amount of free
  2931. * room, don't allocate a chunk. A good mount is
  2932. * less than 80% utilized of the chunks we have allocated,
  2933. * or more than 256MB free
  2934. */
  2935. if (num_allocated + alloc_bytes + 256 * 1024 * 1024 < num_bytes)
  2936. return 0;
  2937. if (num_allocated + alloc_bytes < div_factor(num_bytes, 8))
  2938. return 0;
  2939. thresh = btrfs_super_total_bytes(&root->fs_info->super_copy);
  2940. /* 256MB or 5% of the FS */
  2941. thresh = max_t(u64, 256 * 1024 * 1024, div_factor_fine(thresh, 5));
  2942. if (num_bytes > thresh && sinfo->bytes_used < div_factor(num_bytes, 3))
  2943. return 0;
  2944. return 1;
  2945. }
  2946. static int do_chunk_alloc(struct btrfs_trans_handle *trans,
  2947. struct btrfs_root *extent_root, u64 alloc_bytes,
  2948. u64 flags, int force)
  2949. {
  2950. struct btrfs_space_info *space_info;
  2951. struct btrfs_fs_info *fs_info = extent_root->fs_info;
  2952. int wait_for_alloc = 0;
  2953. int ret = 0;
  2954. flags = btrfs_reduce_alloc_profile(extent_root, flags);
  2955. space_info = __find_space_info(extent_root->fs_info, flags);
  2956. if (!space_info) {
  2957. ret = update_space_info(extent_root->fs_info, flags,
  2958. 0, 0, &space_info);
  2959. BUG_ON(ret);
  2960. }
  2961. BUG_ON(!space_info);
  2962. again:
  2963. spin_lock(&space_info->lock);
  2964. if (space_info->force_alloc)
  2965. force = space_info->force_alloc;
  2966. if (space_info->full) {
  2967. spin_unlock(&space_info->lock);
  2968. return 0;
  2969. }
  2970. if (!should_alloc_chunk(extent_root, space_info, alloc_bytes, force)) {
  2971. spin_unlock(&space_info->lock);
  2972. return 0;
  2973. } else if (space_info->chunk_alloc) {
  2974. wait_for_alloc = 1;
  2975. } else {
  2976. space_info->chunk_alloc = 1;
  2977. }
  2978. spin_unlock(&space_info->lock);
  2979. mutex_lock(&fs_info->chunk_mutex);
  2980. /*
  2981. * The chunk_mutex is held throughout the entirety of a chunk
  2982. * allocation, so once we've acquired the chunk_mutex we know that the
  2983. * other guy is done and we need to recheck and see if we should
  2984. * allocate.
  2985. */
  2986. if (wait_for_alloc) {
  2987. mutex_unlock(&fs_info->chunk_mutex);
  2988. wait_for_alloc = 0;
  2989. goto again;
  2990. }
  2991. /*
  2992. * If we have mixed data/metadata chunks we want to make sure we keep
  2993. * allocating mixed chunks instead of individual chunks.
  2994. */
  2995. if (btrfs_mixed_space_info(space_info))
  2996. flags |= (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA);
  2997. /*
  2998. * if we're doing a data chunk, go ahead and make sure that
  2999. * we keep a reasonable number of metadata chunks allocated in the
  3000. * FS as well.
  3001. */
  3002. if (flags & BTRFS_BLOCK_GROUP_DATA && fs_info->metadata_ratio) {
  3003. fs_info->data_chunk_allocations++;
  3004. if (!(fs_info->data_chunk_allocations %
  3005. fs_info->metadata_ratio))
  3006. force_metadata_allocation(fs_info);
  3007. }
  3008. ret = btrfs_alloc_chunk(trans, extent_root, flags);
  3009. spin_lock(&space_info->lock);
  3010. if (ret)
  3011. space_info->full = 1;
  3012. else
  3013. ret = 1;
  3014. space_info->force_alloc = CHUNK_ALLOC_NO_FORCE;
  3015. space_info->chunk_alloc = 0;
  3016. spin_unlock(&space_info->lock);
  3017. mutex_unlock(&extent_root->fs_info->chunk_mutex);
  3018. return ret;
  3019. }
  3020. /*
  3021. * shrink metadata reservation for delalloc
  3022. */
  3023. static int shrink_delalloc(struct btrfs_trans_handle *trans,
  3024. struct btrfs_root *root, u64 to_reclaim, int sync)
  3025. {
  3026. struct btrfs_block_rsv *block_rsv;
  3027. struct btrfs_space_info *space_info;
  3028. u64 reserved;
  3029. u64 max_reclaim;
  3030. u64 reclaimed = 0;
  3031. long time_left;
  3032. int nr_pages = (2 * 1024 * 1024) >> PAGE_CACHE_SHIFT;
  3033. int loops = 0;
  3034. unsigned long progress;
  3035. block_rsv = &root->fs_info->delalloc_block_rsv;
  3036. space_info = block_rsv->space_info;
  3037. smp_mb();
  3038. reserved = space_info->bytes_reserved;
  3039. progress = space_info->reservation_progress;
  3040. if (reserved == 0)
  3041. return 0;
  3042. max_reclaim = min(reserved, to_reclaim);
  3043. while (loops < 1024) {
  3044. /* have the flusher threads jump in and do some IO */
  3045. smp_mb();
  3046. nr_pages = min_t(unsigned long, nr_pages,
  3047. root->fs_info->delalloc_bytes >> PAGE_CACHE_SHIFT);
  3048. writeback_inodes_sb_nr_if_idle(root->fs_info->sb, nr_pages);
  3049. spin_lock(&space_info->lock);
  3050. if (reserved > space_info->bytes_reserved)
  3051. reclaimed += reserved - space_info->bytes_reserved;
  3052. reserved = space_info->bytes_reserved;
  3053. spin_unlock(&space_info->lock);
  3054. loops++;
  3055. if (reserved == 0 || reclaimed >= max_reclaim)
  3056. break;
  3057. if (trans && trans->transaction->blocked)
  3058. return -EAGAIN;
  3059. time_left = schedule_timeout_interruptible(1);
  3060. /* We were interrupted, exit */
  3061. if (time_left)
  3062. break;
  3063. /* we've kicked the IO a few times, if anything has been freed,
  3064. * exit. There is no sense in looping here for a long time
  3065. * when we really need to commit the transaction, or there are
  3066. * just too many writers without enough free space
  3067. */
  3068. if (loops > 3) {
  3069. smp_mb();
  3070. if (progress != space_info->reservation_progress)
  3071. break;
  3072. }
  3073. }
  3074. return reclaimed >= to_reclaim;
  3075. }
  3076. /*
  3077. * Retries tells us how many times we've called reserve_metadata_bytes. The
  3078. * idea is if this is the first call (retries == 0) then we will add to our
  3079. * reserved count if we can't make the allocation in order to hold our place
  3080. * while we go and try and free up space. That way for retries > 1 we don't try
  3081. * and add space, we just check to see if the amount of unused space is >= the
  3082. * total space, meaning that our reservation is valid.
  3083. *
  3084. * However if we don't intend to retry this reservation, pass -1 as retries so
  3085. * that it short circuits this logic.
  3086. */
  3087. static int reserve_metadata_bytes(struct btrfs_trans_handle *trans,
  3088. struct btrfs_root *root,
  3089. struct btrfs_block_rsv *block_rsv,
  3090. u64 orig_bytes, int flush)
  3091. {
  3092. struct btrfs_space_info *space_info = block_rsv->space_info;
  3093. u64 unused;
  3094. u64 num_bytes = orig_bytes;
  3095. int retries = 0;
  3096. int ret = 0;
  3097. bool reserved = false;
  3098. bool committed = false;
  3099. again:
  3100. ret = -ENOSPC;
  3101. if (reserved)
  3102. num_bytes = 0;
  3103. spin_lock(&space_info->lock);
  3104. unused = space_info->bytes_used + space_info->bytes_reserved +
  3105. space_info->bytes_pinned + space_info->bytes_readonly +
  3106. space_info->bytes_may_use;
  3107. /*
  3108. * The idea here is that we've not already over-reserved the block group
  3109. * then we can go ahead and save our reservation first and then start
  3110. * flushing if we need to. Otherwise if we've already overcommitted
  3111. * lets start flushing stuff first and then come back and try to make
  3112. * our reservation.
  3113. */
  3114. if (unused <= space_info->total_bytes) {
  3115. unused = space_info->total_bytes - unused;
  3116. if (unused >= num_bytes) {
  3117. if (!reserved)
  3118. space_info->bytes_reserved += orig_bytes;
  3119. ret = 0;
  3120. } else {
  3121. /*
  3122. * Ok set num_bytes to orig_bytes since we aren't
  3123. * overocmmitted, this way we only try and reclaim what
  3124. * we need.
  3125. */
  3126. num_bytes = orig_bytes;
  3127. }
  3128. } else {
  3129. /*
  3130. * Ok we're over committed, set num_bytes to the overcommitted
  3131. * amount plus the amount of bytes that we need for this
  3132. * reservation.
  3133. */
  3134. num_bytes = unused - space_info->total_bytes +
  3135. (orig_bytes * (retries + 1));
  3136. }
  3137. /*
  3138. * Couldn't make our reservation, save our place so while we're trying
  3139. * to reclaim space we can actually use it instead of somebody else
  3140. * stealing it from us.
  3141. */
  3142. if (ret && !reserved) {
  3143. space_info->bytes_reserved += orig_bytes;
  3144. reserved = true;
  3145. }
  3146. spin_unlock(&space_info->lock);
  3147. if (!ret)
  3148. return 0;
  3149. if (!flush)
  3150. goto out;
  3151. /*
  3152. * We do synchronous shrinking since we don't actually unreserve
  3153. * metadata until after the IO is completed.
  3154. */
  3155. ret = shrink_delalloc(trans, root, num_bytes, 1);
  3156. if (ret > 0)
  3157. return 0;
  3158. else if (ret < 0)
  3159. goto out;
  3160. /*
  3161. * So if we were overcommitted it's possible that somebody else flushed
  3162. * out enough space and we simply didn't have enough space to reclaim,
  3163. * so go back around and try again.
  3164. */
  3165. if (retries < 2) {
  3166. retries++;
  3167. goto again;
  3168. }
  3169. spin_lock(&space_info->lock);
  3170. /*
  3171. * Not enough space to be reclaimed, don't bother committing the
  3172. * transaction.
  3173. */
  3174. if (space_info->bytes_pinned < orig_bytes)
  3175. ret = -ENOSPC;
  3176. spin_unlock(&space_info->lock);
  3177. if (ret)
  3178. goto out;
  3179. ret = -EAGAIN;
  3180. if (trans || committed)
  3181. goto out;
  3182. ret = -ENOSPC;
  3183. trans = btrfs_join_transaction(root, 1);
  3184. if (IS_ERR(trans))
  3185. goto out;
  3186. ret = btrfs_commit_transaction(trans, root);
  3187. if (!ret) {
  3188. trans = NULL;
  3189. committed = true;
  3190. goto again;
  3191. }
  3192. out:
  3193. if (reserved) {
  3194. spin_lock(&space_info->lock);
  3195. space_info->bytes_reserved -= orig_bytes;
  3196. spin_unlock(&space_info->lock);
  3197. }
  3198. return ret;
  3199. }
  3200. static struct btrfs_block_rsv *get_block_rsv(struct btrfs_trans_handle *trans,
  3201. struct btrfs_root *root)
  3202. {
  3203. struct btrfs_block_rsv *block_rsv;
  3204. if (root->ref_cows)
  3205. block_rsv = trans->block_rsv;
  3206. else
  3207. block_rsv = root->block_rsv;
  3208. if (!block_rsv)
  3209. block_rsv = &root->fs_info->empty_block_rsv;
  3210. return block_rsv;
  3211. }
  3212. static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv,
  3213. u64 num_bytes)
  3214. {
  3215. int ret = -ENOSPC;
  3216. spin_lock(&block_rsv->lock);
  3217. if (block_rsv->reserved >= num_bytes) {
  3218. block_rsv->reserved -= num_bytes;
  3219. if (block_rsv->reserved < block_rsv->size)
  3220. block_rsv->full = 0;
  3221. ret = 0;
  3222. }
  3223. spin_unlock(&block_rsv->lock);
  3224. return ret;
  3225. }
  3226. static void block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv,
  3227. u64 num_bytes, int update_size)
  3228. {
  3229. spin_lock(&block_rsv->lock);
  3230. block_rsv->reserved += num_bytes;
  3231. if (update_size)
  3232. block_rsv->size += num_bytes;
  3233. else if (block_rsv->reserved >= block_rsv->size)
  3234. block_rsv->full = 1;
  3235. spin_unlock(&block_rsv->lock);
  3236. }
  3237. void block_rsv_release_bytes(struct btrfs_block_rsv *block_rsv,
  3238. struct btrfs_block_rsv *dest, u64 num_bytes)
  3239. {
  3240. struct btrfs_space_info *space_info = block_rsv->space_info;
  3241. spin_lock(&block_rsv->lock);
  3242. if (num_bytes == (u64)-1)
  3243. num_bytes = block_rsv->size;
  3244. block_rsv->size -= num_bytes;
  3245. if (block_rsv->reserved >= block_rsv->size) {
  3246. num_bytes = block_rsv->reserved - block_rsv->size;
  3247. block_rsv->reserved = block_rsv->size;
  3248. block_rsv->full = 1;
  3249. } else {
  3250. num_bytes = 0;
  3251. }
  3252. spin_unlock(&block_rsv->lock);
  3253. if (num_bytes > 0) {
  3254. if (dest) {
  3255. spin_lock(&dest->lock);
  3256. if (!dest->full) {
  3257. u64 bytes_to_add;
  3258. bytes_to_add = dest->size - dest->reserved;
  3259. bytes_to_add = min(num_bytes, bytes_to_add);
  3260. dest->reserved += bytes_to_add;
  3261. if (dest->reserved >= dest->size)
  3262. dest->full = 1;
  3263. num_bytes -= bytes_to_add;
  3264. }
  3265. spin_unlock(&dest->lock);
  3266. }
  3267. if (num_bytes) {
  3268. spin_lock(&space_info->lock);
  3269. space_info->bytes_reserved -= num_bytes;
  3270. space_info->reservation_progress++;
  3271. spin_unlock(&space_info->lock);
  3272. }
  3273. }
  3274. }
  3275. static int block_rsv_migrate_bytes(struct btrfs_block_rsv *src,
  3276. struct btrfs_block_rsv *dst, u64 num_bytes)
  3277. {
  3278. int ret;
  3279. ret = block_rsv_use_bytes(src, num_bytes);
  3280. if (ret)
  3281. return ret;
  3282. block_rsv_add_bytes(dst, num_bytes, 1);
  3283. return 0;
  3284. }
  3285. void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv)
  3286. {
  3287. memset(rsv, 0, sizeof(*rsv));
  3288. spin_lock_init(&rsv->lock);
  3289. atomic_set(&rsv->usage, 1);
  3290. rsv->priority = 6;
  3291. INIT_LIST_HEAD(&rsv->list);
  3292. }
  3293. struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_root *root)
  3294. {
  3295. struct btrfs_block_rsv *block_rsv;
  3296. struct btrfs_fs_info *fs_info = root->fs_info;
  3297. block_rsv = kmalloc(sizeof(*block_rsv), GFP_NOFS);
  3298. if (!block_rsv)
  3299. return NULL;
  3300. btrfs_init_block_rsv(block_rsv);
  3301. block_rsv->space_info = __find_space_info(fs_info,
  3302. BTRFS_BLOCK_GROUP_METADATA);
  3303. return block_rsv;
  3304. }
  3305. void btrfs_free_block_rsv(struct btrfs_root *root,
  3306. struct btrfs_block_rsv *rsv)
  3307. {
  3308. if (rsv && atomic_dec_and_test(&rsv->usage)) {
  3309. btrfs_block_rsv_release(root, rsv, (u64)-1);
  3310. if (!rsv->durable)
  3311. kfree(rsv);
  3312. }
  3313. }
  3314. /*
  3315. * make the block_rsv struct be able to capture freed space.
  3316. * the captured space will re-add to the the block_rsv struct
  3317. * after transaction commit
  3318. */
  3319. void btrfs_add_durable_block_rsv(struct btrfs_fs_info *fs_info,
  3320. struct btrfs_block_rsv *block_rsv)
  3321. {
  3322. block_rsv->durable = 1;
  3323. mutex_lock(&fs_info->durable_block_rsv_mutex);
  3324. list_add_tail(&block_rsv->list, &fs_info->durable_block_rsv_list);
  3325. mutex_unlock(&fs_info->durable_block_rsv_mutex);
  3326. }
  3327. int btrfs_block_rsv_add(struct btrfs_trans_handle *trans,
  3328. struct btrfs_root *root,
  3329. struct btrfs_block_rsv *block_rsv,
  3330. u64 num_bytes)
  3331. {
  3332. int ret;
  3333. if (num_bytes == 0)
  3334. return 0;
  3335. ret = reserve_metadata_bytes(trans, root, block_rsv, num_bytes, 1);
  3336. if (!ret) {
  3337. block_rsv_add_bytes(block_rsv, num_bytes, 1);
  3338. return 0;
  3339. }
  3340. return ret;
  3341. }
  3342. int btrfs_block_rsv_check(struct btrfs_trans_handle *trans,
  3343. struct btrfs_root *root,
  3344. struct btrfs_block_rsv *block_rsv,
  3345. u64 min_reserved, int min_factor)
  3346. {
  3347. u64 num_bytes = 0;
  3348. int commit_trans = 0;
  3349. int ret = -ENOSPC;
  3350. if (!block_rsv)
  3351. return 0;
  3352. spin_lock(&block_rsv->lock);
  3353. if (min_factor > 0)
  3354. num_bytes = div_factor(block_rsv->size, min_factor);
  3355. if (min_reserved > num_bytes)
  3356. num_bytes = min_reserved;
  3357. if (block_rsv->reserved >= num_bytes) {
  3358. ret = 0;
  3359. } else {
  3360. num_bytes -= block_rsv->reserved;
  3361. if (block_rsv->durable &&
  3362. block_rsv->freed[0] + block_rsv->freed[1] >= num_bytes)
  3363. commit_trans = 1;
  3364. }
  3365. spin_unlock(&block_rsv->lock);
  3366. if (!ret)
  3367. return 0;
  3368. if (block_rsv->refill_used) {
  3369. ret = reserve_metadata_bytes(trans, root, block_rsv,
  3370. num_bytes, 0);
  3371. if (!ret) {
  3372. block_rsv_add_bytes(block_rsv, num_bytes, 0);
  3373. return 0;
  3374. }
  3375. }
  3376. if (commit_trans) {
  3377. if (trans)
  3378. return -EAGAIN;
  3379. trans = btrfs_join_transaction(root, 1);
  3380. BUG_ON(IS_ERR(trans));
  3381. ret = btrfs_commit_transaction(trans, root);
  3382. return 0;
  3383. }
  3384. return -ENOSPC;
  3385. }
  3386. int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src_rsv,
  3387. struct btrfs_block_rsv *dst_rsv,
  3388. u64 num_bytes)
  3389. {
  3390. return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
  3391. }
  3392. void btrfs_block_rsv_release(struct btrfs_root *root,
  3393. struct btrfs_block_rsv *block_rsv,
  3394. u64 num_bytes)
  3395. {
  3396. struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
  3397. if (global_rsv->full || global_rsv == block_rsv ||
  3398. block_rsv->space_info != global_rsv->space_info)
  3399. global_rsv = NULL;
  3400. block_rsv_release_bytes(block_rsv, global_rsv, num_bytes);
  3401. }
  3402. /*
  3403. * helper to calculate size of global block reservation.
  3404. * the desired value is sum of space used by extent tree,
  3405. * checksum tree and root tree
  3406. */
  3407. static u64 calc_global_metadata_size(struct btrfs_fs_info *fs_info)
  3408. {
  3409. struct btrfs_space_info *sinfo;
  3410. u64 num_bytes;
  3411. u64 meta_used;
  3412. u64 data_used;
  3413. int csum_size = btrfs_super_csum_size(&fs_info->super_copy);
  3414. #if 0
  3415. /*
  3416. * per tree used space accounting can be inaccuracy, so we
  3417. * can't rely on it.
  3418. */
  3419. spin_lock(&fs_info->extent_root->accounting_lock);
  3420. num_bytes = btrfs_root_used(&fs_info->extent_root->root_item);
  3421. spin_unlock(&fs_info->extent_root->accounting_lock);
  3422. spin_lock(&fs_info->csum_root->accounting_lock);
  3423. num_bytes += btrfs_root_used(&fs_info->csum_root->root_item);
  3424. spin_unlock(&fs_info->csum_root->accounting_lock);
  3425. spin_lock(&fs_info->tree_root->accounting_lock);
  3426. num_bytes += btrfs_root_used(&fs_info->tree_root->root_item);
  3427. spin_unlock(&fs_info->tree_root->accounting_lock);
  3428. #endif
  3429. sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_DATA);
  3430. spin_lock(&sinfo->lock);
  3431. data_used = sinfo->bytes_used;
  3432. spin_unlock(&sinfo->lock);
  3433. sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
  3434. spin_lock(&sinfo->lock);
  3435. if (sinfo->flags & BTRFS_BLOCK_GROUP_DATA)
  3436. data_used = 0;
  3437. meta_used = sinfo->bytes_used;
  3438. spin_unlock(&sinfo->lock);
  3439. num_bytes = (data_used >> fs_info->sb->s_blocksize_bits) *
  3440. csum_size * 2;
  3441. num_bytes += div64_u64(data_used + meta_used, 50);
  3442. if (num_bytes * 3 > meta_used)
  3443. num_bytes = div64_u64(meta_used, 3);
  3444. return ALIGN(num_bytes, fs_info->extent_root->leafsize << 10);
  3445. }
  3446. static void update_global_block_rsv(struct btrfs_fs_info *fs_info)
  3447. {
  3448. struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv;
  3449. struct btrfs_space_info *sinfo = block_rsv->space_info;
  3450. u64 num_bytes;
  3451. num_bytes = calc_global_metadata_size(fs_info);
  3452. spin_lock(&block_rsv->lock);
  3453. spin_lock(&sinfo->lock);
  3454. block_rsv->size = num_bytes;
  3455. num_bytes = sinfo->bytes_used + sinfo->bytes_pinned +
  3456. sinfo->bytes_reserved + sinfo->bytes_readonly +
  3457. sinfo->bytes_may_use;
  3458. if (sinfo->total_bytes > num_bytes) {
  3459. num_bytes = sinfo->total_bytes - num_bytes;
  3460. block_rsv->reserved += num_bytes;
  3461. sinfo->bytes_reserved += num_bytes;
  3462. }
  3463. if (block_rsv->reserved >= block_rsv->size) {
  3464. num_bytes = block_rsv->reserved - block_rsv->size;
  3465. sinfo->bytes_reserved -= num_bytes;
  3466. sinfo->reservation_progress++;
  3467. block_rsv->reserved = block_rsv->size;
  3468. block_rsv->full = 1;
  3469. }
  3470. #if 0
  3471. printk(KERN_INFO"global block rsv size %llu reserved %llu\n",
  3472. block_rsv->size, block_rsv->reserved);
  3473. #endif
  3474. spin_unlock(&sinfo->lock);
  3475. spin_unlock(&block_rsv->lock);
  3476. }
  3477. static void init_global_block_rsv(struct btrfs_fs_info *fs_info)
  3478. {
  3479. struct btrfs_space_info *space_info;
  3480. space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
  3481. fs_info->chunk_block_rsv.space_info = space_info;
  3482. fs_info->chunk_block_rsv.priority = 10;
  3483. space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
  3484. fs_info->global_block_rsv.space_info = space_info;
  3485. fs_info->global_block_rsv.priority = 10;
  3486. fs_info->global_block_rsv.refill_used = 1;
  3487. fs_info->delalloc_block_rsv.space_info = space_info;
  3488. fs_info->trans_block_rsv.space_info = space_info;
  3489. fs_info->empty_block_rsv.space_info = space_info;
  3490. fs_info->empty_block_rsv.priority = 10;
  3491. fs_info->extent_root->block_rsv = &fs_info->global_block_rsv;
  3492. fs_info->csum_root->block_rsv = &fs_info->global_block_rsv;
  3493. fs_info->dev_root->block_rsv = &fs_info->global_block_rsv;
  3494. fs_info->tree_root->block_rsv = &fs_info->global_block_rsv;
  3495. fs_info->chunk_root->block_rsv = &fs_info->chunk_block_rsv;
  3496. btrfs_add_durable_block_rsv(fs_info, &fs_info->global_block_rsv);
  3497. btrfs_add_durable_block_rsv(fs_info, &fs_info->delalloc_block_rsv);
  3498. update_global_block_rsv(fs_info);
  3499. }
  3500. static void release_global_block_rsv(struct btrfs_fs_info *fs_info)
  3501. {
  3502. block_rsv_release_bytes(&fs_info->global_block_rsv, NULL, (u64)-1);
  3503. WARN_ON(fs_info->delalloc_block_rsv.size > 0);
  3504. WARN_ON(fs_info->delalloc_block_rsv.reserved > 0);
  3505. WARN_ON(fs_info->trans_block_rsv.size > 0);
  3506. WARN_ON(fs_info->trans_block_rsv.reserved > 0);
  3507. WARN_ON(fs_info->chunk_block_rsv.size > 0);
  3508. WARN_ON(fs_info->chunk_block_rsv.reserved > 0);
  3509. }
  3510. static u64 calc_trans_metadata_size(struct btrfs_root *root, int num_items)
  3511. {
  3512. return (root->leafsize + root->nodesize * (BTRFS_MAX_LEVEL - 1)) *
  3513. 3 * num_items;
  3514. }
  3515. int btrfs_trans_reserve_metadata(struct btrfs_trans_handle *trans,
  3516. struct btrfs_root *root,
  3517. int num_items)
  3518. {
  3519. u64 num_bytes;
  3520. int ret;
  3521. if (num_items == 0 || root->fs_info->chunk_root == root)
  3522. return 0;
  3523. num_bytes = calc_trans_metadata_size(root, num_items);
  3524. ret = btrfs_block_rsv_add(trans, root, &root->fs_info->trans_block_rsv,
  3525. num_bytes);
  3526. if (!ret) {
  3527. trans->bytes_reserved += num_bytes;
  3528. trans->block_rsv = &root->fs_info->trans_block_rsv;
  3529. }
  3530. return ret;
  3531. }
  3532. void btrfs_trans_release_metadata(struct btrfs_trans_handle *trans,
  3533. struct btrfs_root *root)
  3534. {
  3535. if (!trans->bytes_reserved)
  3536. return;
  3537. BUG_ON(trans->block_rsv != &root->fs_info->trans_block_rsv);
  3538. btrfs_block_rsv_release(root, trans->block_rsv,
  3539. trans->bytes_reserved);
  3540. trans->bytes_reserved = 0;
  3541. }
  3542. int btrfs_orphan_reserve_metadata(struct btrfs_trans_handle *trans,
  3543. struct inode *inode)
  3544. {
  3545. struct btrfs_root *root = BTRFS_I(inode)->root;
  3546. struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
  3547. struct btrfs_block_rsv *dst_rsv = root->orphan_block_rsv;
  3548. /*
  3549. * one for deleting orphan item, one for updating inode and
  3550. * two for calling btrfs_truncate_inode_items.
  3551. *
  3552. * btrfs_truncate_inode_items is a delete operation, it frees
  3553. * more space than it uses in most cases. So two units of
  3554. * metadata space should be enough for calling it many times.
  3555. * If all of the metadata space is used, we can commit
  3556. * transaction and use space it freed.
  3557. */
  3558. u64 num_bytes = calc_trans_metadata_size(root, 4);
  3559. return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
  3560. }
  3561. void btrfs_orphan_release_metadata(struct inode *inode)
  3562. {
  3563. struct btrfs_root *root = BTRFS_I(inode)->root;
  3564. u64 num_bytes = calc_trans_metadata_size(root, 4);
  3565. btrfs_block_rsv_release(root, root->orphan_block_rsv, num_bytes);
  3566. }
  3567. int btrfs_snap_reserve_metadata(struct btrfs_trans_handle *trans,
  3568. struct btrfs_pending_snapshot *pending)
  3569. {
  3570. struct btrfs_root *root = pending->root;
  3571. struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
  3572. struct btrfs_block_rsv *dst_rsv = &pending->block_rsv;
  3573. /*
  3574. * two for root back/forward refs, two for directory entries
  3575. * and one for root of the snapshot.
  3576. */
  3577. u64 num_bytes = calc_trans_metadata_size(root, 5);
  3578. dst_rsv->space_info = src_rsv->space_info;
  3579. return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
  3580. }
  3581. static u64 calc_csum_metadata_size(struct inode *inode, u64 num_bytes)
  3582. {
  3583. return num_bytes >>= 3;
  3584. }
  3585. int btrfs_delalloc_reserve_metadata(struct inode *inode, u64 num_bytes)
  3586. {
  3587. struct btrfs_root *root = BTRFS_I(inode)->root;
  3588. struct btrfs_block_rsv *block_rsv = &root->fs_info->delalloc_block_rsv;
  3589. u64 to_reserve;
  3590. int nr_extents;
  3591. int reserved_extents;
  3592. int ret;
  3593. if (btrfs_transaction_in_commit(root->fs_info))
  3594. schedule_timeout(1);
  3595. num_bytes = ALIGN(num_bytes, root->sectorsize);
  3596. nr_extents = atomic_read(&BTRFS_I(inode)->outstanding_extents) + 1;
  3597. reserved_extents = atomic_read(&BTRFS_I(inode)->reserved_extents);
  3598. if (nr_extents > reserved_extents) {
  3599. nr_extents -= reserved_extents;
  3600. to_reserve = calc_trans_metadata_size(root, nr_extents);
  3601. } else {
  3602. nr_extents = 0;
  3603. to_reserve = 0;
  3604. }
  3605. to_reserve += calc_csum_metadata_size(inode, num_bytes);
  3606. ret = reserve_metadata_bytes(NULL, root, block_rsv, to_reserve, 1);
  3607. if (ret)
  3608. return ret;
  3609. atomic_add(nr_extents, &BTRFS_I(inode)->reserved_extents);
  3610. atomic_inc(&BTRFS_I(inode)->outstanding_extents);
  3611. block_rsv_add_bytes(block_rsv, to_reserve, 1);
  3612. if (block_rsv->size > 512 * 1024 * 1024)
  3613. shrink_delalloc(NULL, root, to_reserve, 0);
  3614. return 0;
  3615. }
  3616. void btrfs_delalloc_release_metadata(struct inode *inode, u64 num_bytes)
  3617. {
  3618. struct btrfs_root *root = BTRFS_I(inode)->root;
  3619. u64 to_free;
  3620. int nr_extents;
  3621. int reserved_extents;
  3622. num_bytes = ALIGN(num_bytes, root->sectorsize);
  3623. atomic_dec(&BTRFS_I(inode)->outstanding_extents);
  3624. WARN_ON(atomic_read(&BTRFS_I(inode)->outstanding_extents) < 0);
  3625. reserved_extents = atomic_read(&BTRFS_I(inode)->reserved_extents);
  3626. do {
  3627. int old, new;
  3628. nr_extents = atomic_read(&BTRFS_I(inode)->outstanding_extents);
  3629. if (nr_extents >= reserved_extents) {
  3630. nr_extents = 0;
  3631. break;
  3632. }
  3633. old = reserved_extents;
  3634. nr_extents = reserved_extents - nr_extents;
  3635. new = reserved_extents - nr_extents;
  3636. old = atomic_cmpxchg(&BTRFS_I(inode)->reserved_extents,
  3637. reserved_extents, new);
  3638. if (likely(old == reserved_extents))
  3639. break;
  3640. reserved_extents = old;
  3641. } while (1);
  3642. to_free = calc_csum_metadata_size(inode, num_bytes);
  3643. if (nr_extents > 0)
  3644. to_free += calc_trans_metadata_size(root, nr_extents);
  3645. btrfs_block_rsv_release(root, &root->fs_info->delalloc_block_rsv,
  3646. to_free);
  3647. }
  3648. int btrfs_delalloc_reserve_space(struct inode *inode, u64 num_bytes)
  3649. {
  3650. int ret;
  3651. ret = btrfs_check_data_free_space(inode, num_bytes);
  3652. if (ret)
  3653. return ret;
  3654. ret = btrfs_delalloc_reserve_metadata(inode, num_bytes);
  3655. if (ret) {
  3656. btrfs_free_reserved_data_space(inode, num_bytes);
  3657. return ret;
  3658. }
  3659. return 0;
  3660. }
  3661. void btrfs_delalloc_release_space(struct inode *inode, u64 num_bytes)
  3662. {
  3663. btrfs_delalloc_release_metadata(inode, num_bytes);
  3664. btrfs_free_reserved_data_space(inode, num_bytes);
  3665. }
  3666. static int update_block_group(struct btrfs_trans_handle *trans,
  3667. struct btrfs_root *root,
  3668. u64 bytenr, u64 num_bytes, int alloc)
  3669. {
  3670. struct btrfs_block_group_cache *cache = NULL;
  3671. struct btrfs_fs_info *info = root->fs_info;
  3672. u64 total = num_bytes;
  3673. u64 old_val;
  3674. u64 byte_in_group;
  3675. int factor;
  3676. /* block accounting for super block */
  3677. spin_lock(&info->delalloc_lock);
  3678. old_val = btrfs_super_bytes_used(&info->super_copy);
  3679. if (alloc)
  3680. old_val += num_bytes;
  3681. else
  3682. old_val -= num_bytes;
  3683. btrfs_set_super_bytes_used(&info->super_copy, old_val);
  3684. spin_unlock(&info->delalloc_lock);
  3685. while (total) {
  3686. cache = btrfs_lookup_block_group(info, bytenr);
  3687. if (!cache)
  3688. return -1;
  3689. if (cache->flags & (BTRFS_BLOCK_GROUP_DUP |
  3690. BTRFS_BLOCK_GROUP_RAID1 |
  3691. BTRFS_BLOCK_GROUP_RAID10))
  3692. factor = 2;
  3693. else
  3694. factor = 1;
  3695. /*
  3696. * If this block group has free space cache written out, we
  3697. * need to make sure to load it if we are removing space. This
  3698. * is because we need the unpinning stage to actually add the
  3699. * space back to the block group, otherwise we will leak space.
  3700. */
  3701. if (!alloc && cache->cached == BTRFS_CACHE_NO)
  3702. cache_block_group(cache, trans, NULL, 1);
  3703. byte_in_group = bytenr - cache->key.objectid;
  3704. WARN_ON(byte_in_group > cache->key.offset);
  3705. spin_lock(&cache->space_info->lock);
  3706. spin_lock(&cache->lock);
  3707. if (btrfs_super_cache_generation(&info->super_copy) != 0 &&
  3708. cache->disk_cache_state < BTRFS_DC_CLEAR)
  3709. cache->disk_cache_state = BTRFS_DC_CLEAR;
  3710. cache->dirty = 1;
  3711. old_val = btrfs_block_group_used(&cache->item);
  3712. num_bytes = min(total, cache->key.offset - byte_in_group);
  3713. if (alloc) {
  3714. old_val += num_bytes;
  3715. btrfs_set_block_group_used(&cache->item, old_val);
  3716. cache->reserved -= num_bytes;
  3717. cache->space_info->bytes_reserved -= num_bytes;
  3718. cache->space_info->reservation_progress++;
  3719. cache->space_info->bytes_used += num_bytes;
  3720. cache->space_info->disk_used += num_bytes * factor;
  3721. spin_unlock(&cache->lock);
  3722. spin_unlock(&cache->space_info->lock);
  3723. } else {
  3724. old_val -= num_bytes;
  3725. btrfs_set_block_group_used(&cache->item, old_val);
  3726. cache->pinned += num_bytes;
  3727. cache->space_info->bytes_pinned += num_bytes;
  3728. cache->space_info->bytes_used -= num_bytes;
  3729. cache->space_info->disk_used -= num_bytes * factor;
  3730. spin_unlock(&cache->lock);
  3731. spin_unlock(&cache->space_info->lock);
  3732. set_extent_dirty(info->pinned_extents,
  3733. bytenr, bytenr + num_bytes - 1,
  3734. GFP_NOFS | __GFP_NOFAIL);
  3735. }
  3736. btrfs_put_block_group(cache);
  3737. total -= num_bytes;
  3738. bytenr += num_bytes;
  3739. }
  3740. return 0;
  3741. }
  3742. static u64 first_logical_byte(struct btrfs_root *root, u64 search_start)
  3743. {
  3744. struct btrfs_block_group_cache *cache;
  3745. u64 bytenr;
  3746. cache = btrfs_lookup_first_block_group(root->fs_info, search_start);
  3747. if (!cache)
  3748. return 0;
  3749. bytenr = cache->key.objectid;
  3750. btrfs_put_block_group(cache);
  3751. return bytenr;
  3752. }
  3753. static int pin_down_extent(struct btrfs_root *root,
  3754. struct btrfs_block_group_cache *cache,
  3755. u64 bytenr, u64 num_bytes, int reserved)
  3756. {
  3757. spin_lock(&cache->space_info->lock);
  3758. spin_lock(&cache->lock);
  3759. cache->pinned += num_bytes;
  3760. cache->space_info->bytes_pinned += num_bytes;
  3761. if (reserved) {
  3762. cache->reserved -= num_bytes;
  3763. cache->space_info->bytes_reserved -= num_bytes;
  3764. cache->space_info->reservation_progress++;
  3765. }
  3766. spin_unlock(&cache->lock);
  3767. spin_unlock(&cache->space_info->lock);
  3768. set_extent_dirty(root->fs_info->pinned_extents, bytenr,
  3769. bytenr + num_bytes - 1, GFP_NOFS | __GFP_NOFAIL);
  3770. return 0;
  3771. }
  3772. /*
  3773. * this function must be called within transaction
  3774. */
  3775. int btrfs_pin_extent(struct btrfs_root *root,
  3776. u64 bytenr, u64 num_bytes, int reserved)
  3777. {
  3778. struct btrfs_block_group_cache *cache;
  3779. cache = btrfs_lookup_block_group(root->fs_info, bytenr);
  3780. BUG_ON(!cache);
  3781. pin_down_extent(root, cache, bytenr, num_bytes, reserved);
  3782. btrfs_put_block_group(cache);
  3783. return 0;
  3784. }
  3785. /*
  3786. * update size of reserved extents. this function may return -EAGAIN
  3787. * if 'reserve' is true or 'sinfo' is false.
  3788. */
  3789. int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache,
  3790. u64 num_bytes, int reserve, int sinfo)
  3791. {
  3792. int ret = 0;
  3793. if (sinfo) {
  3794. struct btrfs_space_info *space_info = cache->space_info;
  3795. spin_lock(&space_info->lock);
  3796. spin_lock(&cache->lock);
  3797. if (reserve) {
  3798. if (cache->ro) {
  3799. ret = -EAGAIN;
  3800. } else {
  3801. cache->reserved += num_bytes;
  3802. space_info->bytes_reserved += num_bytes;
  3803. }
  3804. } else {
  3805. if (cache->ro)
  3806. space_info->bytes_readonly += num_bytes;
  3807. cache->reserved -= num_bytes;
  3808. space_info->bytes_reserved -= num_bytes;
  3809. space_info->reservation_progress++;
  3810. }
  3811. spin_unlock(&cache->lock);
  3812. spin_unlock(&space_info->lock);
  3813. } else {
  3814. spin_lock(&cache->lock);
  3815. if (cache->ro) {
  3816. ret = -EAGAIN;
  3817. } else {
  3818. if (reserve)
  3819. cache->reserved += num_bytes;
  3820. else
  3821. cache->reserved -= num_bytes;
  3822. }
  3823. spin_unlock(&cache->lock);
  3824. }
  3825. return ret;
  3826. }
  3827. int btrfs_prepare_extent_commit(struct btrfs_trans_handle *trans,
  3828. struct btrfs_root *root)
  3829. {
  3830. struct btrfs_fs_info *fs_info = root->fs_info;
  3831. struct btrfs_caching_control *next;
  3832. struct btrfs_caching_control *caching_ctl;
  3833. struct btrfs_block_group_cache *cache;
  3834. down_write(&fs_info->extent_commit_sem);
  3835. list_for_each_entry_safe(caching_ctl, next,
  3836. &fs_info->caching_block_groups, list) {
  3837. cache = caching_ctl->block_group;
  3838. if (block_group_cache_done(cache)) {
  3839. cache->last_byte_to_unpin = (u64)-1;
  3840. list_del_init(&caching_ctl->list);
  3841. put_caching_control(caching_ctl);
  3842. } else {
  3843. cache->last_byte_to_unpin = caching_ctl->progress;
  3844. }
  3845. }
  3846. if (fs_info->pinned_extents == &fs_info->freed_extents[0])
  3847. fs_info->pinned_extents = &fs_info->freed_extents[1];
  3848. else
  3849. fs_info->pinned_extents = &fs_info->freed_extents[0];
  3850. up_write(&fs_info->extent_commit_sem);
  3851. update_global_block_rsv(fs_info);
  3852. return 0;
  3853. }
  3854. static int unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
  3855. {
  3856. struct btrfs_fs_info *fs_info = root->fs_info;
  3857. struct btrfs_block_group_cache *cache = NULL;
  3858. u64 len;
  3859. while (start <= end) {
  3860. if (!cache ||
  3861. start >= cache->key.objectid + cache->key.offset) {
  3862. if (cache)
  3863. btrfs_put_block_group(cache);
  3864. cache = btrfs_lookup_block_group(fs_info, start);
  3865. BUG_ON(!cache);
  3866. }
  3867. len = cache->key.objectid + cache->key.offset - start;
  3868. len = min(len, end + 1 - start);
  3869. if (start < cache->last_byte_to_unpin) {
  3870. len = min(len, cache->last_byte_to_unpin - start);
  3871. btrfs_add_free_space(cache, start, len);
  3872. }
  3873. start += len;
  3874. spin_lock(&cache->space_info->lock);
  3875. spin_lock(&cache->lock);
  3876. cache->pinned -= len;
  3877. cache->space_info->bytes_pinned -= len;
  3878. if (cache->ro) {
  3879. cache->space_info->bytes_readonly += len;
  3880. } else if (cache->reserved_pinned > 0) {
  3881. len = min(len, cache->reserved_pinned);
  3882. cache->reserved_pinned -= len;
  3883. cache->space_info->bytes_reserved += len;
  3884. }
  3885. spin_unlock(&cache->lock);
  3886. spin_unlock(&cache->space_info->lock);
  3887. }
  3888. if (cache)
  3889. btrfs_put_block_group(cache);
  3890. return 0;
  3891. }
  3892. int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans,
  3893. struct btrfs_root *root)
  3894. {
  3895. struct btrfs_fs_info *fs_info = root->fs_info;
  3896. struct extent_io_tree *unpin;
  3897. struct btrfs_block_rsv *block_rsv;
  3898. struct btrfs_block_rsv *next_rsv;
  3899. u64 start;
  3900. u64 end;
  3901. int idx;
  3902. int ret;
  3903. if (fs_info->pinned_extents == &fs_info->freed_extents[0])
  3904. unpin = &fs_info->freed_extents[1];
  3905. else
  3906. unpin = &fs_info->freed_extents[0];
  3907. while (1) {
  3908. ret = find_first_extent_bit(unpin, 0, &start, &end,
  3909. EXTENT_DIRTY);
  3910. if (ret)
  3911. break;
  3912. if (btrfs_test_opt(root, DISCARD))
  3913. ret = btrfs_discard_extent(root, start,
  3914. end + 1 - start, NULL);
  3915. clear_extent_dirty(unpin, start, end, GFP_NOFS);
  3916. unpin_extent_range(root, start, end);
  3917. cond_resched();
  3918. }
  3919. mutex_lock(&fs_info->durable_block_rsv_mutex);
  3920. list_for_each_entry_safe(block_rsv, next_rsv,
  3921. &fs_info->durable_block_rsv_list, list) {
  3922. idx = trans->transid & 0x1;
  3923. if (block_rsv->freed[idx] > 0) {
  3924. block_rsv_add_bytes(block_rsv,
  3925. block_rsv->freed[idx], 0);
  3926. block_rsv->freed[idx] = 0;
  3927. }
  3928. if (atomic_read(&block_rsv->usage) == 0) {
  3929. btrfs_block_rsv_release(root, block_rsv, (u64)-1);
  3930. if (block_rsv->freed[0] == 0 &&
  3931. block_rsv->freed[1] == 0) {
  3932. list_del_init(&block_rsv->list);
  3933. kfree(block_rsv);
  3934. }
  3935. } else {
  3936. btrfs_block_rsv_release(root, block_rsv, 0);
  3937. }
  3938. }
  3939. mutex_unlock(&fs_info->durable_block_rsv_mutex);
  3940. return 0;
  3941. }
  3942. static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
  3943. struct btrfs_root *root,
  3944. u64 bytenr, u64 num_bytes, u64 parent,
  3945. u64 root_objectid, u64 owner_objectid,
  3946. u64 owner_offset, int refs_to_drop,
  3947. struct btrfs_delayed_extent_op *extent_op)
  3948. {
  3949. struct btrfs_key key;
  3950. struct btrfs_path *path;
  3951. struct btrfs_fs_info *info = root->fs_info;
  3952. struct btrfs_root *extent_root = info->extent_root;
  3953. struct extent_buffer *leaf;
  3954. struct btrfs_extent_item *ei;
  3955. struct btrfs_extent_inline_ref *iref;
  3956. int ret;
  3957. int is_data;
  3958. int extent_slot = 0;
  3959. int found_extent = 0;
  3960. int num_to_del = 1;
  3961. u32 item_size;
  3962. u64 refs;
  3963. path = btrfs_alloc_path();
  3964. if (!path)
  3965. return -ENOMEM;
  3966. path->reada = 1;
  3967. path->leave_spinning = 1;
  3968. is_data = owner_objectid >= BTRFS_FIRST_FREE_OBJECTID;
  3969. BUG_ON(!is_data && refs_to_drop != 1);
  3970. ret = lookup_extent_backref(trans, extent_root, path, &iref,
  3971. bytenr, num_bytes, parent,
  3972. root_objectid, owner_objectid,
  3973. owner_offset);
  3974. if (ret == 0) {
  3975. extent_slot = path->slots[0];
  3976. while (extent_slot >= 0) {
  3977. btrfs_item_key_to_cpu(path->nodes[0], &key,
  3978. extent_slot);
  3979. if (key.objectid != bytenr)
  3980. break;
  3981. if (key.type == BTRFS_EXTENT_ITEM_KEY &&
  3982. key.offset == num_bytes) {
  3983. found_extent = 1;
  3984. break;
  3985. }
  3986. if (path->slots[0] - extent_slot > 5)
  3987. break;
  3988. extent_slot--;
  3989. }
  3990. #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
  3991. item_size = btrfs_item_size_nr(path->nodes[0], extent_slot);
  3992. if (found_extent && item_size < sizeof(*ei))
  3993. found_extent = 0;
  3994. #endif
  3995. if (!found_extent) {
  3996. BUG_ON(iref);
  3997. ret = remove_extent_backref(trans, extent_root, path,
  3998. NULL, refs_to_drop,
  3999. is_data);
  4000. BUG_ON(ret);
  4001. btrfs_release_path(extent_root, path);
  4002. path->leave_spinning = 1;
  4003. key.objectid = bytenr;
  4004. key.type = BTRFS_EXTENT_ITEM_KEY;
  4005. key.offset = num_bytes;
  4006. ret = btrfs_search_slot(trans, extent_root,
  4007. &key, path, -1, 1);
  4008. if (ret) {
  4009. printk(KERN_ERR "umm, got %d back from search"
  4010. ", was looking for %llu\n", ret,
  4011. (unsigned long long)bytenr);
  4012. btrfs_print_leaf(extent_root, path->nodes[0]);
  4013. }
  4014. BUG_ON(ret);
  4015. extent_slot = path->slots[0];
  4016. }
  4017. } else {
  4018. btrfs_print_leaf(extent_root, path->nodes[0]);
  4019. WARN_ON(1);
  4020. printk(KERN_ERR "btrfs unable to find ref byte nr %llu "
  4021. "parent %llu root %llu owner %llu offset %llu\n",
  4022. (unsigned long long)bytenr,
  4023. (unsigned long long)parent,
  4024. (unsigned long long)root_objectid,
  4025. (unsigned long long)owner_objectid,
  4026. (unsigned long long)owner_offset);
  4027. }
  4028. leaf = path->nodes[0];
  4029. item_size = btrfs_item_size_nr(leaf, extent_slot);
  4030. #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
  4031. if (item_size < sizeof(*ei)) {
  4032. BUG_ON(found_extent || extent_slot != path->slots[0]);
  4033. ret = convert_extent_item_v0(trans, extent_root, path,
  4034. owner_objectid, 0);
  4035. BUG_ON(ret < 0);
  4036. btrfs_release_path(extent_root, path);
  4037. path->leave_spinning = 1;
  4038. key.objectid = bytenr;
  4039. key.type = BTRFS_EXTENT_ITEM_KEY;
  4040. key.offset = num_bytes;
  4041. ret = btrfs_search_slot(trans, extent_root, &key, path,
  4042. -1, 1);
  4043. if (ret) {
  4044. printk(KERN_ERR "umm, got %d back from search"
  4045. ", was looking for %llu\n", ret,
  4046. (unsigned long long)bytenr);
  4047. btrfs_print_leaf(extent_root, path->nodes[0]);
  4048. }
  4049. BUG_ON(ret);
  4050. extent_slot = path->slots[0];
  4051. leaf = path->nodes[0];
  4052. item_size = btrfs_item_size_nr(leaf, extent_slot);
  4053. }
  4054. #endif
  4055. BUG_ON(item_size < sizeof(*ei));
  4056. ei = btrfs_item_ptr(leaf, extent_slot,
  4057. struct btrfs_extent_item);
  4058. if (owner_objectid < BTRFS_FIRST_FREE_OBJECTID) {
  4059. struct btrfs_tree_block_info *bi;
  4060. BUG_ON(item_size < sizeof(*ei) + sizeof(*bi));
  4061. bi = (struct btrfs_tree_block_info *)(ei + 1);
  4062. WARN_ON(owner_objectid != btrfs_tree_block_level(leaf, bi));
  4063. }
  4064. refs = btrfs_extent_refs(leaf, ei);
  4065. BUG_ON(refs < refs_to_drop);
  4066. refs -= refs_to_drop;
  4067. if (refs > 0) {
  4068. if (extent_op)
  4069. __run_delayed_extent_op(extent_op, leaf, ei);
  4070. /*
  4071. * In the case of inline back ref, reference count will
  4072. * be updated by remove_extent_backref
  4073. */
  4074. if (iref) {
  4075. BUG_ON(!found_extent);
  4076. } else {
  4077. btrfs_set_extent_refs(leaf, ei, refs);
  4078. btrfs_mark_buffer_dirty(leaf);
  4079. }
  4080. if (found_extent) {
  4081. ret = remove_extent_backref(trans, extent_root, path,
  4082. iref, refs_to_drop,
  4083. is_data);
  4084. BUG_ON(ret);
  4085. }
  4086. } else {
  4087. if (found_extent) {
  4088. BUG_ON(is_data && refs_to_drop !=
  4089. extent_data_ref_count(root, path, iref));
  4090. if (iref) {
  4091. BUG_ON(path->slots[0] != extent_slot);
  4092. } else {
  4093. BUG_ON(path->slots[0] != extent_slot + 1);
  4094. path->slots[0] = extent_slot;
  4095. num_to_del = 2;
  4096. }
  4097. }
  4098. ret = btrfs_del_items(trans, extent_root, path, path->slots[0],
  4099. num_to_del);
  4100. BUG_ON(ret);
  4101. btrfs_release_path(extent_root, path);
  4102. if (is_data) {
  4103. ret = btrfs_del_csums(trans, root, bytenr, num_bytes);
  4104. BUG_ON(ret);
  4105. } else {
  4106. invalidate_mapping_pages(info->btree_inode->i_mapping,
  4107. bytenr >> PAGE_CACHE_SHIFT,
  4108. (bytenr + num_bytes - 1) >> PAGE_CACHE_SHIFT);
  4109. }
  4110. ret = update_block_group(trans, root, bytenr, num_bytes, 0);
  4111. BUG_ON(ret);
  4112. }
  4113. btrfs_free_path(path);
  4114. return ret;
  4115. }
  4116. /*
  4117. * when we free an block, it is possible (and likely) that we free the last
  4118. * delayed ref for that extent as well. This searches the delayed ref tree for
  4119. * a given extent, and if there are no other delayed refs to be processed, it
  4120. * removes it from the tree.
  4121. */
  4122. static noinline int check_ref_cleanup(struct btrfs_trans_handle *trans,
  4123. struct btrfs_root *root, u64 bytenr)
  4124. {
  4125. struct btrfs_delayed_ref_head *head;
  4126. struct btrfs_delayed_ref_root *delayed_refs;
  4127. struct btrfs_delayed_ref_node *ref;
  4128. struct rb_node *node;
  4129. int ret = 0;
  4130. delayed_refs = &trans->transaction->delayed_refs;
  4131. spin_lock(&delayed_refs->lock);
  4132. head = btrfs_find_delayed_ref_head(trans, bytenr);
  4133. if (!head)
  4134. goto out;
  4135. node = rb_prev(&head->node.rb_node);
  4136. if (!node)
  4137. goto out;
  4138. ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);
  4139. /* there are still entries for this ref, we can't drop it */
  4140. if (ref->bytenr == bytenr)
  4141. goto out;
  4142. if (head->extent_op) {
  4143. if (!head->must_insert_reserved)
  4144. goto out;
  4145. kfree(head->extent_op);
  4146. head->extent_op = NULL;
  4147. }
  4148. /*
  4149. * waiting for the lock here would deadlock. If someone else has it
  4150. * locked they are already in the process of dropping it anyway
  4151. */
  4152. if (!mutex_trylock(&head->mutex))
  4153. goto out;
  4154. /*
  4155. * at this point we have a head with no other entries. Go
  4156. * ahead and process it.
  4157. */
  4158. head->node.in_tree = 0;
  4159. rb_erase(&head->node.rb_node, &delayed_refs->root);
  4160. delayed_refs->num_entries--;
  4161. /*
  4162. * we don't take a ref on the node because we're removing it from the
  4163. * tree, so we just steal the ref the tree was holding.
  4164. */
  4165. delayed_refs->num_heads--;
  4166. if (list_empty(&head->cluster))
  4167. delayed_refs->num_heads_ready--;
  4168. list_del_init(&head->cluster);
  4169. spin_unlock(&delayed_refs->lock);
  4170. BUG_ON(head->extent_op);
  4171. if (head->must_insert_reserved)
  4172. ret = 1;
  4173. mutex_unlock(&head->mutex);
  4174. btrfs_put_delayed_ref(&head->node);
  4175. return ret;
  4176. out:
  4177. spin_unlock(&delayed_refs->lock);
  4178. return 0;
  4179. }
  4180. void btrfs_free_tree_block(struct btrfs_trans_handle *trans,
  4181. struct btrfs_root *root,
  4182. struct extent_buffer *buf,
  4183. u64 parent, int last_ref)
  4184. {
  4185. struct btrfs_block_rsv *block_rsv;
  4186. struct btrfs_block_group_cache *cache = NULL;
  4187. int ret;
  4188. if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
  4189. ret = btrfs_add_delayed_tree_ref(trans, buf->start, buf->len,
  4190. parent, root->root_key.objectid,
  4191. btrfs_header_level(buf),
  4192. BTRFS_DROP_DELAYED_REF, NULL);
  4193. BUG_ON(ret);
  4194. }
  4195. if (!last_ref)
  4196. return;
  4197. block_rsv = get_block_rsv(trans, root);
  4198. cache = btrfs_lookup_block_group(root->fs_info, buf->start);
  4199. if (block_rsv->space_info != cache->space_info)
  4200. goto out;
  4201. if (btrfs_header_generation(buf) == trans->transid) {
  4202. if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
  4203. ret = check_ref_cleanup(trans, root, buf->start);
  4204. if (!ret)
  4205. goto pin;
  4206. }
  4207. if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) {
  4208. pin_down_extent(root, cache, buf->start, buf->len, 1);
  4209. goto pin;
  4210. }
  4211. WARN_ON(test_bit(EXTENT_BUFFER_DIRTY, &buf->bflags));
  4212. btrfs_add_free_space(cache, buf->start, buf->len);
  4213. ret = btrfs_update_reserved_bytes(cache, buf->len, 0, 0);
  4214. if (ret == -EAGAIN) {
  4215. /* block group became read-only */
  4216. btrfs_update_reserved_bytes(cache, buf->len, 0, 1);
  4217. goto out;
  4218. }
  4219. ret = 1;
  4220. spin_lock(&block_rsv->lock);
  4221. if (block_rsv->reserved < block_rsv->size) {
  4222. block_rsv->reserved += buf->len;
  4223. ret = 0;
  4224. }
  4225. spin_unlock(&block_rsv->lock);
  4226. if (ret) {
  4227. spin_lock(&cache->space_info->lock);
  4228. cache->space_info->bytes_reserved -= buf->len;
  4229. cache->space_info->reservation_progress++;
  4230. spin_unlock(&cache->space_info->lock);
  4231. }
  4232. goto out;
  4233. }
  4234. pin:
  4235. if (block_rsv->durable && !cache->ro) {
  4236. ret = 0;
  4237. spin_lock(&cache->lock);
  4238. if (!cache->ro) {
  4239. cache->reserved_pinned += buf->len;
  4240. ret = 1;
  4241. }
  4242. spin_unlock(&cache->lock);
  4243. if (ret) {
  4244. spin_lock(&block_rsv->lock);
  4245. block_rsv->freed[trans->transid & 0x1] += buf->len;
  4246. spin_unlock(&block_rsv->lock);
  4247. }
  4248. }
  4249. out:
  4250. /*
  4251. * Deleting the buffer, clear the corrupt flag since it doesn't matter
  4252. * anymore.
  4253. */
  4254. clear_bit(EXTENT_BUFFER_CORRUPT, &buf->bflags);
  4255. btrfs_put_block_group(cache);
  4256. }
  4257. int btrfs_free_extent(struct btrfs_trans_handle *trans,
  4258. struct btrfs_root *root,
  4259. u64 bytenr, u64 num_bytes, u64 parent,
  4260. u64 root_objectid, u64 owner, u64 offset)
  4261. {
  4262. int ret;
  4263. /*
  4264. * tree log blocks never actually go into the extent allocation
  4265. * tree, just update pinning info and exit early.
  4266. */
  4267. if (root_objectid == BTRFS_TREE_LOG_OBJECTID) {
  4268. WARN_ON(owner >= BTRFS_FIRST_FREE_OBJECTID);
  4269. /* unlocks the pinned mutex */
  4270. btrfs_pin_extent(root, bytenr, num_bytes, 1);
  4271. ret = 0;
  4272. } else if (owner < BTRFS_FIRST_FREE_OBJECTID) {
  4273. ret = btrfs_add_delayed_tree_ref(trans, bytenr, num_bytes,
  4274. parent, root_objectid, (int)owner,
  4275. BTRFS_DROP_DELAYED_REF, NULL);
  4276. BUG_ON(ret);
  4277. } else {
  4278. ret = btrfs_add_delayed_data_ref(trans, bytenr, num_bytes,
  4279. parent, root_objectid, owner,
  4280. offset, BTRFS_DROP_DELAYED_REF, NULL);
  4281. BUG_ON(ret);
  4282. }
  4283. return ret;
  4284. }
  4285. static u64 stripe_align(struct btrfs_root *root, u64 val)
  4286. {
  4287. u64 mask = ((u64)root->stripesize - 1);
  4288. u64 ret = (val + mask) & ~mask;
  4289. return ret;
  4290. }
  4291. /*
  4292. * when we wait for progress in the block group caching, its because
  4293. * our allocation attempt failed at least once. So, we must sleep
  4294. * and let some progress happen before we try again.
  4295. *
  4296. * This function will sleep at least once waiting for new free space to
  4297. * show up, and then it will check the block group free space numbers
  4298. * for our min num_bytes. Another option is to have it go ahead
  4299. * and look in the rbtree for a free extent of a given size, but this
  4300. * is a good start.
  4301. */
  4302. static noinline int
  4303. wait_block_group_cache_progress(struct btrfs_block_group_cache *cache,
  4304. u64 num_bytes)
  4305. {
  4306. struct btrfs_caching_control *caching_ctl;
  4307. DEFINE_WAIT(wait);
  4308. caching_ctl = get_caching_control(cache);
  4309. if (!caching_ctl)
  4310. return 0;
  4311. wait_event(caching_ctl->wait, block_group_cache_done(cache) ||
  4312. (cache->free_space_ctl->free_space >= num_bytes));
  4313. put_caching_control(caching_ctl);
  4314. return 0;
  4315. }
  4316. static noinline int
  4317. wait_block_group_cache_done(struct btrfs_block_group_cache *cache)
  4318. {
  4319. struct btrfs_caching_control *caching_ctl;
  4320. DEFINE_WAIT(wait);
  4321. caching_ctl = get_caching_control(cache);
  4322. if (!caching_ctl)
  4323. return 0;
  4324. wait_event(caching_ctl->wait, block_group_cache_done(cache));
  4325. put_caching_control(caching_ctl);
  4326. return 0;
  4327. }
  4328. static int get_block_group_index(struct btrfs_block_group_cache *cache)
  4329. {
  4330. int index;
  4331. if (cache->flags & BTRFS_BLOCK_GROUP_RAID10)
  4332. index = 0;
  4333. else if (cache->flags & BTRFS_BLOCK_GROUP_RAID1)
  4334. index = 1;
  4335. else if (cache->flags & BTRFS_BLOCK_GROUP_DUP)
  4336. index = 2;
  4337. else if (cache->flags & BTRFS_BLOCK_GROUP_RAID0)
  4338. index = 3;
  4339. else
  4340. index = 4;
  4341. return index;
  4342. }
  4343. enum btrfs_loop_type {
  4344. LOOP_FIND_IDEAL = 0,
  4345. LOOP_CACHING_NOWAIT = 1,
  4346. LOOP_CACHING_WAIT = 2,
  4347. LOOP_ALLOC_CHUNK = 3,
  4348. LOOP_NO_EMPTY_SIZE = 4,
  4349. };
  4350. /*
  4351. * walks the btree of allocated extents and find a hole of a given size.
  4352. * The key ins is changed to record the hole:
  4353. * ins->objectid == block start
  4354. * ins->flags = BTRFS_EXTENT_ITEM_KEY
  4355. * ins->offset == number of blocks
  4356. * Any available blocks before search_start are skipped.
  4357. */
  4358. static noinline int find_free_extent(struct btrfs_trans_handle *trans,
  4359. struct btrfs_root *orig_root,
  4360. u64 num_bytes, u64 empty_size,
  4361. u64 search_start, u64 search_end,
  4362. u64 hint_byte, struct btrfs_key *ins,
  4363. int data)
  4364. {
  4365. int ret = 0;
  4366. struct btrfs_root *root = orig_root->fs_info->extent_root;
  4367. struct btrfs_free_cluster *last_ptr = NULL;
  4368. struct btrfs_block_group_cache *block_group = NULL;
  4369. int empty_cluster = 2 * 1024 * 1024;
  4370. int allowed_chunk_alloc = 0;
  4371. int done_chunk_alloc = 0;
  4372. struct btrfs_space_info *space_info;
  4373. int last_ptr_loop = 0;
  4374. int loop = 0;
  4375. int index = 0;
  4376. bool found_uncached_bg = false;
  4377. bool failed_cluster_refill = false;
  4378. bool failed_alloc = false;
  4379. bool use_cluster = true;
  4380. u64 ideal_cache_percent = 0;
  4381. u64 ideal_cache_offset = 0;
  4382. WARN_ON(num_bytes < root->sectorsize);
  4383. btrfs_set_key_type(ins, BTRFS_EXTENT_ITEM_KEY);
  4384. ins->objectid = 0;
  4385. ins->offset = 0;
  4386. space_info = __find_space_info(root->fs_info, data);
  4387. if (!space_info) {
  4388. printk(KERN_ERR "No space info for %d\n", data);
  4389. return -ENOSPC;
  4390. }
  4391. /*
  4392. * If the space info is for both data and metadata it means we have a
  4393. * small filesystem and we can't use the clustering stuff.
  4394. */
  4395. if (btrfs_mixed_space_info(space_info))
  4396. use_cluster = false;
  4397. if (orig_root->ref_cows || empty_size)
  4398. allowed_chunk_alloc = 1;
  4399. if (data & BTRFS_BLOCK_GROUP_METADATA && use_cluster) {
  4400. last_ptr = &root->fs_info->meta_alloc_cluster;
  4401. if (!btrfs_test_opt(root, SSD))
  4402. empty_cluster = 64 * 1024;
  4403. }
  4404. if ((data & BTRFS_BLOCK_GROUP_DATA) && use_cluster &&
  4405. btrfs_test_opt(root, SSD)) {
  4406. last_ptr = &root->fs_info->data_alloc_cluster;
  4407. }
  4408. if (last_ptr) {
  4409. spin_lock(&last_ptr->lock);
  4410. if (last_ptr->block_group)
  4411. hint_byte = last_ptr->window_start;
  4412. spin_unlock(&last_ptr->lock);
  4413. }
  4414. search_start = max(search_start, first_logical_byte(root, 0));
  4415. search_start = max(search_start, hint_byte);
  4416. if (!last_ptr)
  4417. empty_cluster = 0;
  4418. if (search_start == hint_byte) {
  4419. ideal_cache:
  4420. block_group = btrfs_lookup_block_group(root->fs_info,
  4421. search_start);
  4422. /*
  4423. * we don't want to use the block group if it doesn't match our
  4424. * allocation bits, or if its not cached.
  4425. *
  4426. * However if we are re-searching with an ideal block group
  4427. * picked out then we don't care that the block group is cached.
  4428. */
  4429. if (block_group && block_group_bits(block_group, data) &&
  4430. (block_group->cached != BTRFS_CACHE_NO ||
  4431. search_start == ideal_cache_offset)) {
  4432. down_read(&space_info->groups_sem);
  4433. if (list_empty(&block_group->list) ||
  4434. block_group->ro) {
  4435. /*
  4436. * someone is removing this block group,
  4437. * we can't jump into the have_block_group
  4438. * target because our list pointers are not
  4439. * valid
  4440. */
  4441. btrfs_put_block_group(block_group);
  4442. up_read(&space_info->groups_sem);
  4443. } else {
  4444. index = get_block_group_index(block_group);
  4445. goto have_block_group;
  4446. }
  4447. } else if (block_group) {
  4448. btrfs_put_block_group(block_group);
  4449. }
  4450. }
  4451. search:
  4452. down_read(&space_info->groups_sem);
  4453. list_for_each_entry(block_group, &space_info->block_groups[index],
  4454. list) {
  4455. u64 offset;
  4456. int cached;
  4457. btrfs_get_block_group(block_group);
  4458. search_start = block_group->key.objectid;
  4459. /*
  4460. * this can happen if we end up cycling through all the
  4461. * raid types, but we want to make sure we only allocate
  4462. * for the proper type.
  4463. */
  4464. if (!block_group_bits(block_group, data)) {
  4465. u64 extra = BTRFS_BLOCK_GROUP_DUP |
  4466. BTRFS_BLOCK_GROUP_RAID1 |
  4467. BTRFS_BLOCK_GROUP_RAID10;
  4468. /*
  4469. * if they asked for extra copies and this block group
  4470. * doesn't provide them, bail. This does allow us to
  4471. * fill raid0 from raid1.
  4472. */
  4473. if ((data & extra) && !(block_group->flags & extra))
  4474. goto loop;
  4475. }
  4476. have_block_group:
  4477. if (unlikely(block_group->cached == BTRFS_CACHE_NO)) {
  4478. u64 free_percent;
  4479. ret = cache_block_group(block_group, trans,
  4480. orig_root, 1);
  4481. if (block_group->cached == BTRFS_CACHE_FINISHED)
  4482. goto have_block_group;
  4483. free_percent = btrfs_block_group_used(&block_group->item);
  4484. free_percent *= 100;
  4485. free_percent = div64_u64(free_percent,
  4486. block_group->key.offset);
  4487. free_percent = 100 - free_percent;
  4488. if (free_percent > ideal_cache_percent &&
  4489. likely(!block_group->ro)) {
  4490. ideal_cache_offset = block_group->key.objectid;
  4491. ideal_cache_percent = free_percent;
  4492. }
  4493. /*
  4494. * We only want to start kthread caching if we are at
  4495. * the point where we will wait for caching to make
  4496. * progress, or if our ideal search is over and we've
  4497. * found somebody to start caching.
  4498. */
  4499. if (loop > LOOP_CACHING_NOWAIT ||
  4500. (loop > LOOP_FIND_IDEAL &&
  4501. atomic_read(&space_info->caching_threads) < 2)) {
  4502. ret = cache_block_group(block_group, trans,
  4503. orig_root, 0);
  4504. BUG_ON(ret);
  4505. }
  4506. found_uncached_bg = true;
  4507. /*
  4508. * If loop is set for cached only, try the next block
  4509. * group.
  4510. */
  4511. if (loop == LOOP_FIND_IDEAL)
  4512. goto loop;
  4513. }
  4514. cached = block_group_cache_done(block_group);
  4515. if (unlikely(!cached))
  4516. found_uncached_bg = true;
  4517. if (unlikely(block_group->ro))
  4518. goto loop;
  4519. /*
  4520. * Ok we want to try and use the cluster allocator, so lets look
  4521. * there, unless we are on LOOP_NO_EMPTY_SIZE, since we will
  4522. * have tried the cluster allocator plenty of times at this
  4523. * point and not have found anything, so we are likely way too
  4524. * fragmented for the clustering stuff to find anything, so lets
  4525. * just skip it and let the allocator find whatever block it can
  4526. * find
  4527. */
  4528. if (last_ptr && loop < LOOP_NO_EMPTY_SIZE) {
  4529. /*
  4530. * the refill lock keeps out other
  4531. * people trying to start a new cluster
  4532. */
  4533. spin_lock(&last_ptr->refill_lock);
  4534. if (last_ptr->block_group &&
  4535. (last_ptr->block_group->ro ||
  4536. !block_group_bits(last_ptr->block_group, data))) {
  4537. offset = 0;
  4538. goto refill_cluster;
  4539. }
  4540. offset = btrfs_alloc_from_cluster(block_group, last_ptr,
  4541. num_bytes, search_start);
  4542. if (offset) {
  4543. /* we have a block, we're done */
  4544. spin_unlock(&last_ptr->refill_lock);
  4545. goto checks;
  4546. }
  4547. spin_lock(&last_ptr->lock);
  4548. /*
  4549. * whoops, this cluster doesn't actually point to
  4550. * this block group. Get a ref on the block
  4551. * group is does point to and try again
  4552. */
  4553. if (!last_ptr_loop && last_ptr->block_group &&
  4554. last_ptr->block_group != block_group) {
  4555. btrfs_put_block_group(block_group);
  4556. block_group = last_ptr->block_group;
  4557. btrfs_get_block_group(block_group);
  4558. spin_unlock(&last_ptr->lock);
  4559. spin_unlock(&last_ptr->refill_lock);
  4560. last_ptr_loop = 1;
  4561. search_start = block_group->key.objectid;
  4562. /*
  4563. * we know this block group is properly
  4564. * in the list because
  4565. * btrfs_remove_block_group, drops the
  4566. * cluster before it removes the block
  4567. * group from the list
  4568. */
  4569. goto have_block_group;
  4570. }
  4571. spin_unlock(&last_ptr->lock);
  4572. refill_cluster:
  4573. /*
  4574. * this cluster didn't work out, free it and
  4575. * start over
  4576. */
  4577. btrfs_return_cluster_to_free_space(NULL, last_ptr);
  4578. last_ptr_loop = 0;
  4579. /* allocate a cluster in this block group */
  4580. ret = btrfs_find_space_cluster(trans, root,
  4581. block_group, last_ptr,
  4582. offset, num_bytes,
  4583. empty_cluster + empty_size);
  4584. if (ret == 0) {
  4585. /*
  4586. * now pull our allocation out of this
  4587. * cluster
  4588. */
  4589. offset = btrfs_alloc_from_cluster(block_group,
  4590. last_ptr, num_bytes,
  4591. search_start);
  4592. if (offset) {
  4593. /* we found one, proceed */
  4594. spin_unlock(&last_ptr->refill_lock);
  4595. goto checks;
  4596. }
  4597. } else if (!cached && loop > LOOP_CACHING_NOWAIT
  4598. && !failed_cluster_refill) {
  4599. spin_unlock(&last_ptr->refill_lock);
  4600. failed_cluster_refill = true;
  4601. wait_block_group_cache_progress(block_group,
  4602. num_bytes + empty_cluster + empty_size);
  4603. goto have_block_group;
  4604. }
  4605. /*
  4606. * at this point we either didn't find a cluster
  4607. * or we weren't able to allocate a block from our
  4608. * cluster. Free the cluster we've been trying
  4609. * to use, and go to the next block group
  4610. */
  4611. btrfs_return_cluster_to_free_space(NULL, last_ptr);
  4612. spin_unlock(&last_ptr->refill_lock);
  4613. goto loop;
  4614. }
  4615. offset = btrfs_find_space_for_alloc(block_group, search_start,
  4616. num_bytes, empty_size);
  4617. /*
  4618. * If we didn't find a chunk, and we haven't failed on this
  4619. * block group before, and this block group is in the middle of
  4620. * caching and we are ok with waiting, then go ahead and wait
  4621. * for progress to be made, and set failed_alloc to true.
  4622. *
  4623. * If failed_alloc is true then we've already waited on this
  4624. * block group once and should move on to the next block group.
  4625. */
  4626. if (!offset && !failed_alloc && !cached &&
  4627. loop > LOOP_CACHING_NOWAIT) {
  4628. wait_block_group_cache_progress(block_group,
  4629. num_bytes + empty_size);
  4630. failed_alloc = true;
  4631. goto have_block_group;
  4632. } else if (!offset) {
  4633. goto loop;
  4634. }
  4635. checks:
  4636. search_start = stripe_align(root, offset);
  4637. /* move on to the next group */
  4638. if (search_start + num_bytes >= search_end) {
  4639. btrfs_add_free_space(block_group, offset, num_bytes);
  4640. goto loop;
  4641. }
  4642. /* move on to the next group */
  4643. if (search_start + num_bytes >
  4644. block_group->key.objectid + block_group->key.offset) {
  4645. btrfs_add_free_space(block_group, offset, num_bytes);
  4646. goto loop;
  4647. }
  4648. ins->objectid = search_start;
  4649. ins->offset = num_bytes;
  4650. if (offset < search_start)
  4651. btrfs_add_free_space(block_group, offset,
  4652. search_start - offset);
  4653. BUG_ON(offset > search_start);
  4654. ret = btrfs_update_reserved_bytes(block_group, num_bytes, 1,
  4655. (data & BTRFS_BLOCK_GROUP_DATA));
  4656. if (ret == -EAGAIN) {
  4657. btrfs_add_free_space(block_group, offset, num_bytes);
  4658. goto loop;
  4659. }
  4660. /* we are all good, lets return */
  4661. ins->objectid = search_start;
  4662. ins->offset = num_bytes;
  4663. if (offset < search_start)
  4664. btrfs_add_free_space(block_group, offset,
  4665. search_start - offset);
  4666. BUG_ON(offset > search_start);
  4667. break;
  4668. loop:
  4669. failed_cluster_refill = false;
  4670. failed_alloc = false;
  4671. BUG_ON(index != get_block_group_index(block_group));
  4672. btrfs_put_block_group(block_group);
  4673. }
  4674. up_read(&space_info->groups_sem);
  4675. if (!ins->objectid && ++index < BTRFS_NR_RAID_TYPES)
  4676. goto search;
  4677. /* LOOP_FIND_IDEAL, only search caching/cached bg's, and don't wait for
  4678. * for them to make caching progress. Also
  4679. * determine the best possible bg to cache
  4680. * LOOP_CACHING_NOWAIT, search partially cached block groups, kicking
  4681. * caching kthreads as we move along
  4682. * LOOP_CACHING_WAIT, search everything, and wait if our bg is caching
  4683. * LOOP_ALLOC_CHUNK, force a chunk allocation and try again
  4684. * LOOP_NO_EMPTY_SIZE, set empty_size and empty_cluster to 0 and try
  4685. * again
  4686. */
  4687. if (!ins->objectid && loop < LOOP_NO_EMPTY_SIZE &&
  4688. (found_uncached_bg || empty_size || empty_cluster ||
  4689. allowed_chunk_alloc)) {
  4690. index = 0;
  4691. if (loop == LOOP_FIND_IDEAL && found_uncached_bg) {
  4692. found_uncached_bg = false;
  4693. loop++;
  4694. if (!ideal_cache_percent &&
  4695. atomic_read(&space_info->caching_threads))
  4696. goto search;
  4697. /*
  4698. * 1 of the following 2 things have happened so far
  4699. *
  4700. * 1) We found an ideal block group for caching that
  4701. * is mostly full and will cache quickly, so we might
  4702. * as well wait for it.
  4703. *
  4704. * 2) We searched for cached only and we didn't find
  4705. * anything, and we didn't start any caching kthreads
  4706. * either, so chances are we will loop through and
  4707. * start a couple caching kthreads, and then come back
  4708. * around and just wait for them. This will be slower
  4709. * because we will have 2 caching kthreads reading at
  4710. * the same time when we could have just started one
  4711. * and waited for it to get far enough to give us an
  4712. * allocation, so go ahead and go to the wait caching
  4713. * loop.
  4714. */
  4715. loop = LOOP_CACHING_WAIT;
  4716. search_start = ideal_cache_offset;
  4717. ideal_cache_percent = 0;
  4718. goto ideal_cache;
  4719. } else if (loop == LOOP_FIND_IDEAL) {
  4720. /*
  4721. * Didn't find a uncached bg, wait on anything we find
  4722. * next.
  4723. */
  4724. loop = LOOP_CACHING_WAIT;
  4725. goto search;
  4726. }
  4727. if (loop < LOOP_CACHING_WAIT) {
  4728. loop++;
  4729. goto search;
  4730. }
  4731. if (loop == LOOP_ALLOC_CHUNK) {
  4732. empty_size = 0;
  4733. empty_cluster = 0;
  4734. }
  4735. if (allowed_chunk_alloc) {
  4736. ret = do_chunk_alloc(trans, root, num_bytes +
  4737. 2 * 1024 * 1024, data,
  4738. CHUNK_ALLOC_LIMITED);
  4739. allowed_chunk_alloc = 0;
  4740. done_chunk_alloc = 1;
  4741. } else if (!done_chunk_alloc &&
  4742. space_info->force_alloc == CHUNK_ALLOC_NO_FORCE) {
  4743. space_info->force_alloc = CHUNK_ALLOC_LIMITED;
  4744. }
  4745. if (loop < LOOP_NO_EMPTY_SIZE) {
  4746. loop++;
  4747. goto search;
  4748. }
  4749. ret = -ENOSPC;
  4750. } else if (!ins->objectid) {
  4751. ret = -ENOSPC;
  4752. }
  4753. /* we found what we needed */
  4754. if (ins->objectid) {
  4755. if (!(data & BTRFS_BLOCK_GROUP_DATA))
  4756. trans->block_group = block_group->key.objectid;
  4757. btrfs_put_block_group(block_group);
  4758. ret = 0;
  4759. }
  4760. return ret;
  4761. }
  4762. static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
  4763. int dump_block_groups)
  4764. {
  4765. struct btrfs_block_group_cache *cache;
  4766. int index = 0;
  4767. spin_lock(&info->lock);
  4768. printk(KERN_INFO "space_info has %llu free, is %sfull\n",
  4769. (unsigned long long)(info->total_bytes - info->bytes_used -
  4770. info->bytes_pinned - info->bytes_reserved -
  4771. info->bytes_readonly),
  4772. (info->full) ? "" : "not ");
  4773. printk(KERN_INFO "space_info total=%llu, used=%llu, pinned=%llu, "
  4774. "reserved=%llu, may_use=%llu, readonly=%llu\n",
  4775. (unsigned long long)info->total_bytes,
  4776. (unsigned long long)info->bytes_used,
  4777. (unsigned long long)info->bytes_pinned,
  4778. (unsigned long long)info->bytes_reserved,
  4779. (unsigned long long)info->bytes_may_use,
  4780. (unsigned long long)info->bytes_readonly);
  4781. spin_unlock(&info->lock);
  4782. if (!dump_block_groups)
  4783. return;
  4784. down_read(&info->groups_sem);
  4785. again:
  4786. list_for_each_entry(cache, &info->block_groups[index], list) {
  4787. spin_lock(&cache->lock);
  4788. printk(KERN_INFO "block group %llu has %llu bytes, %llu used "
  4789. "%llu pinned %llu reserved\n",
  4790. (unsigned long long)cache->key.objectid,
  4791. (unsigned long long)cache->key.offset,
  4792. (unsigned long long)btrfs_block_group_used(&cache->item),
  4793. (unsigned long long)cache->pinned,
  4794. (unsigned long long)cache->reserved);
  4795. btrfs_dump_free_space(cache, bytes);
  4796. spin_unlock(&cache->lock);
  4797. }
  4798. if (++index < BTRFS_NR_RAID_TYPES)
  4799. goto again;
  4800. up_read(&info->groups_sem);
  4801. }
  4802. int btrfs_reserve_extent(struct btrfs_trans_handle *trans,
  4803. struct btrfs_root *root,
  4804. u64 num_bytes, u64 min_alloc_size,
  4805. u64 empty_size, u64 hint_byte,
  4806. u64 search_end, struct btrfs_key *ins,
  4807. u64 data)
  4808. {
  4809. int ret;
  4810. u64 search_start = 0;
  4811. data = btrfs_get_alloc_profile(root, data);
  4812. again:
  4813. /*
  4814. * the only place that sets empty_size is btrfs_realloc_node, which
  4815. * is not called recursively on allocations
  4816. */
  4817. if (empty_size || root->ref_cows)
  4818. ret = do_chunk_alloc(trans, root->fs_info->extent_root,
  4819. num_bytes + 2 * 1024 * 1024, data,
  4820. CHUNK_ALLOC_NO_FORCE);
  4821. WARN_ON(num_bytes < root->sectorsize);
  4822. ret = find_free_extent(trans, root, num_bytes, empty_size,
  4823. search_start, search_end, hint_byte,
  4824. ins, data);
  4825. if (ret == -ENOSPC && num_bytes > min_alloc_size) {
  4826. num_bytes = num_bytes >> 1;
  4827. num_bytes = num_bytes & ~(root->sectorsize - 1);
  4828. num_bytes = max(num_bytes, min_alloc_size);
  4829. do_chunk_alloc(trans, root->fs_info->extent_root,
  4830. num_bytes, data, CHUNK_ALLOC_FORCE);
  4831. goto again;
  4832. }
  4833. if (ret == -ENOSPC && btrfs_test_opt(root, ENOSPC_DEBUG)) {
  4834. struct btrfs_space_info *sinfo;
  4835. sinfo = __find_space_info(root->fs_info, data);
  4836. printk(KERN_ERR "btrfs allocation failed flags %llu, "
  4837. "wanted %llu\n", (unsigned long long)data,
  4838. (unsigned long long)num_bytes);
  4839. dump_space_info(sinfo, num_bytes, 1);
  4840. }
  4841. trace_btrfs_reserved_extent_alloc(root, ins->objectid, ins->offset);
  4842. return ret;
  4843. }
  4844. int btrfs_free_reserved_extent(struct btrfs_root *root, u64 start, u64 len)
  4845. {
  4846. struct btrfs_block_group_cache *cache;
  4847. int ret = 0;
  4848. cache = btrfs_lookup_block_group(root->fs_info, start);
  4849. if (!cache) {
  4850. printk(KERN_ERR "Unable to find block group for %llu\n",
  4851. (unsigned long long)start);
  4852. return -ENOSPC;
  4853. }
  4854. if (btrfs_test_opt(root, DISCARD))
  4855. ret = btrfs_discard_extent(root, start, len, NULL);
  4856. btrfs_add_free_space(cache, start, len);
  4857. btrfs_update_reserved_bytes(cache, len, 0, 1);
  4858. btrfs_put_block_group(cache);
  4859. trace_btrfs_reserved_extent_free(root, start, len);
  4860. return ret;
  4861. }
  4862. static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
  4863. struct btrfs_root *root,
  4864. u64 parent, u64 root_objectid,
  4865. u64 flags, u64 owner, u64 offset,
  4866. struct btrfs_key *ins, int ref_mod)
  4867. {
  4868. int ret;
  4869. struct btrfs_fs_info *fs_info = root->fs_info;
  4870. struct btrfs_extent_item *extent_item;
  4871. struct btrfs_extent_inline_ref *iref;
  4872. struct btrfs_path *path;
  4873. struct extent_buffer *leaf;
  4874. int type;
  4875. u32 size;
  4876. if (parent > 0)
  4877. type = BTRFS_SHARED_DATA_REF_KEY;
  4878. else
  4879. type = BTRFS_EXTENT_DATA_REF_KEY;
  4880. size = sizeof(*extent_item) + btrfs_extent_inline_ref_size(type);
  4881. path = btrfs_alloc_path();
  4882. if (!path)
  4883. return -ENOMEM;
  4884. path->leave_spinning = 1;
  4885. ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
  4886. ins, size);
  4887. BUG_ON(ret);
  4888. leaf = path->nodes[0];
  4889. extent_item = btrfs_item_ptr(leaf, path->slots[0],
  4890. struct btrfs_extent_item);
  4891. btrfs_set_extent_refs(leaf, extent_item, ref_mod);
  4892. btrfs_set_extent_generation(leaf, extent_item, trans->transid);
  4893. btrfs_set_extent_flags(leaf, extent_item,
  4894. flags | BTRFS_EXTENT_FLAG_DATA);
  4895. iref = (struct btrfs_extent_inline_ref *)(extent_item + 1);
  4896. btrfs_set_extent_inline_ref_type(leaf, iref, type);
  4897. if (parent > 0) {
  4898. struct btrfs_shared_data_ref *ref;
  4899. ref = (struct btrfs_shared_data_ref *)(iref + 1);
  4900. btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
  4901. btrfs_set_shared_data_ref_count(leaf, ref, ref_mod);
  4902. } else {
  4903. struct btrfs_extent_data_ref *ref;
  4904. ref = (struct btrfs_extent_data_ref *)(&iref->offset);
  4905. btrfs_set_extent_data_ref_root(leaf, ref, root_objectid);
  4906. btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
  4907. btrfs_set_extent_data_ref_offset(leaf, ref, offset);
  4908. btrfs_set_extent_data_ref_count(leaf, ref, ref_mod);
  4909. }
  4910. btrfs_mark_buffer_dirty(path->nodes[0]);
  4911. btrfs_free_path(path);
  4912. ret = update_block_group(trans, root, ins->objectid, ins->offset, 1);
  4913. if (ret) {
  4914. printk(KERN_ERR "btrfs update block group failed for %llu "
  4915. "%llu\n", (unsigned long long)ins->objectid,
  4916. (unsigned long long)ins->offset);
  4917. BUG();
  4918. }
  4919. return ret;
  4920. }
  4921. static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
  4922. struct btrfs_root *root,
  4923. u64 parent, u64 root_objectid,
  4924. u64 flags, struct btrfs_disk_key *key,
  4925. int level, struct btrfs_key *ins)
  4926. {
  4927. int ret;
  4928. struct btrfs_fs_info *fs_info = root->fs_info;
  4929. struct btrfs_extent_item *extent_item;
  4930. struct btrfs_tree_block_info *block_info;
  4931. struct btrfs_extent_inline_ref *iref;
  4932. struct btrfs_path *path;
  4933. struct extent_buffer *leaf;
  4934. u32 size = sizeof(*extent_item) + sizeof(*block_info) + sizeof(*iref);
  4935. path = btrfs_alloc_path();
  4936. BUG_ON(!path);
  4937. path->leave_spinning = 1;
  4938. ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
  4939. ins, size);
  4940. BUG_ON(ret);
  4941. leaf = path->nodes[0];
  4942. extent_item = btrfs_item_ptr(leaf, path->slots[0],
  4943. struct btrfs_extent_item);
  4944. btrfs_set_extent_refs(leaf, extent_item, 1);
  4945. btrfs_set_extent_generation(leaf, extent_item, trans->transid);
  4946. btrfs_set_extent_flags(leaf, extent_item,
  4947. flags | BTRFS_EXTENT_FLAG_TREE_BLOCK);
  4948. block_info = (struct btrfs_tree_block_info *)(extent_item + 1);
  4949. btrfs_set_tree_block_key(leaf, block_info, key);
  4950. btrfs_set_tree_block_level(leaf, block_info, level);
  4951. iref = (struct btrfs_extent_inline_ref *)(block_info + 1);
  4952. if (parent > 0) {
  4953. BUG_ON(!(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF));
  4954. btrfs_set_extent_inline_ref_type(leaf, iref,
  4955. BTRFS_SHARED_BLOCK_REF_KEY);
  4956. btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
  4957. } else {
  4958. btrfs_set_extent_inline_ref_type(leaf, iref,
  4959. BTRFS_TREE_BLOCK_REF_KEY);
  4960. btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
  4961. }
  4962. btrfs_mark_buffer_dirty(leaf);
  4963. btrfs_free_path(path);
  4964. ret = update_block_group(trans, root, ins->objectid, ins->offset, 1);
  4965. if (ret) {
  4966. printk(KERN_ERR "btrfs update block group failed for %llu "
  4967. "%llu\n", (unsigned long long)ins->objectid,
  4968. (unsigned long long)ins->offset);
  4969. BUG();
  4970. }
  4971. return ret;
  4972. }
  4973. int btrfs_alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
  4974. struct btrfs_root *root,
  4975. u64 root_objectid, u64 owner,
  4976. u64 offset, struct btrfs_key *ins)
  4977. {
  4978. int ret;
  4979. BUG_ON(root_objectid == BTRFS_TREE_LOG_OBJECTID);
  4980. ret = btrfs_add_delayed_data_ref(trans, ins->objectid, ins->offset,
  4981. 0, root_objectid, owner, offset,
  4982. BTRFS_ADD_DELAYED_EXTENT, NULL);
  4983. return ret;
  4984. }
  4985. /*
  4986. * this is used by the tree logging recovery code. It records that
  4987. * an extent has been allocated and makes sure to clear the free
  4988. * space cache bits as well
  4989. */
  4990. int btrfs_alloc_logged_file_extent(struct btrfs_trans_handle *trans,
  4991. struct btrfs_root *root,
  4992. u64 root_objectid, u64 owner, u64 offset,
  4993. struct btrfs_key *ins)
  4994. {
  4995. int ret;
  4996. struct btrfs_block_group_cache *block_group;
  4997. struct btrfs_caching_control *caching_ctl;
  4998. u64 start = ins->objectid;
  4999. u64 num_bytes = ins->offset;
  5000. block_group = btrfs_lookup_block_group(root->fs_info, ins->objectid);
  5001. cache_block_group(block_group, trans, NULL, 0);
  5002. caching_ctl = get_caching_control(block_group);
  5003. if (!caching_ctl) {
  5004. BUG_ON(!block_group_cache_done(block_group));
  5005. ret = btrfs_remove_free_space(block_group, start, num_bytes);
  5006. BUG_ON(ret);
  5007. } else {
  5008. mutex_lock(&caching_ctl->mutex);
  5009. if (start >= caching_ctl->progress) {
  5010. ret = add_excluded_extent(root, start, num_bytes);
  5011. BUG_ON(ret);
  5012. } else if (start + num_bytes <= caching_ctl->progress) {
  5013. ret = btrfs_remove_free_space(block_group,
  5014. start, num_bytes);
  5015. BUG_ON(ret);
  5016. } else {
  5017. num_bytes = caching_ctl->progress - start;
  5018. ret = btrfs_remove_free_space(block_group,
  5019. start, num_bytes);
  5020. BUG_ON(ret);
  5021. start = caching_ctl->progress;
  5022. num_bytes = ins->objectid + ins->offset -
  5023. caching_ctl->progress;
  5024. ret = add_excluded_extent(root, start, num_bytes);
  5025. BUG_ON(ret);
  5026. }
  5027. mutex_unlock(&caching_ctl->mutex);
  5028. put_caching_control(caching_ctl);
  5029. }
  5030. ret = btrfs_update_reserved_bytes(block_group, ins->offset, 1, 1);
  5031. BUG_ON(ret);
  5032. btrfs_put_block_group(block_group);
  5033. ret = alloc_reserved_file_extent(trans, root, 0, root_objectid,
  5034. 0, owner, offset, ins, 1);
  5035. return ret;
  5036. }
  5037. struct extent_buffer *btrfs_init_new_buffer(struct btrfs_trans_handle *trans,
  5038. struct btrfs_root *root,
  5039. u64 bytenr, u32 blocksize,
  5040. int level)
  5041. {
  5042. struct extent_buffer *buf;
  5043. buf = btrfs_find_create_tree_block(root, bytenr, blocksize);
  5044. if (!buf)
  5045. return ERR_PTR(-ENOMEM);
  5046. btrfs_set_header_generation(buf, trans->transid);
  5047. btrfs_set_buffer_lockdep_class(buf, level);
  5048. btrfs_tree_lock(buf);
  5049. clean_tree_block(trans, root, buf);
  5050. btrfs_set_lock_blocking(buf);
  5051. btrfs_set_buffer_uptodate(buf);
  5052. if (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID) {
  5053. /*
  5054. * we allow two log transactions at a time, use different
  5055. * EXENT bit to differentiate dirty pages.
  5056. */
  5057. if (root->log_transid % 2 == 0)
  5058. set_extent_dirty(&root->dirty_log_pages, buf->start,
  5059. buf->start + buf->len - 1, GFP_NOFS);
  5060. else
  5061. set_extent_new(&root->dirty_log_pages, buf->start,
  5062. buf->start + buf->len - 1, GFP_NOFS);
  5063. } else {
  5064. set_extent_dirty(&trans->transaction->dirty_pages, buf->start,
  5065. buf->start + buf->len - 1, GFP_NOFS);
  5066. }
  5067. trans->blocks_used++;
  5068. /* this returns a buffer locked for blocking */
  5069. return buf;
  5070. }
  5071. static struct btrfs_block_rsv *
  5072. use_block_rsv(struct btrfs_trans_handle *trans,
  5073. struct btrfs_root *root, u32 blocksize)
  5074. {
  5075. struct btrfs_block_rsv *block_rsv;
  5076. struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
  5077. int ret;
  5078. block_rsv = get_block_rsv(trans, root);
  5079. if (block_rsv->size == 0) {
  5080. ret = reserve_metadata_bytes(trans, root, block_rsv,
  5081. blocksize, 0);
  5082. /*
  5083. * If we couldn't reserve metadata bytes try and use some from
  5084. * the global reserve.
  5085. */
  5086. if (ret && block_rsv != global_rsv) {
  5087. ret = block_rsv_use_bytes(global_rsv, blocksize);
  5088. if (!ret)
  5089. return global_rsv;
  5090. return ERR_PTR(ret);
  5091. } else if (ret) {
  5092. return ERR_PTR(ret);
  5093. }
  5094. return block_rsv;
  5095. }
  5096. ret = block_rsv_use_bytes(block_rsv, blocksize);
  5097. if (!ret)
  5098. return block_rsv;
  5099. if (ret) {
  5100. WARN_ON(1);
  5101. ret = reserve_metadata_bytes(trans, root, block_rsv, blocksize,
  5102. 0);
  5103. if (!ret) {
  5104. spin_lock(&block_rsv->lock);
  5105. block_rsv->size += blocksize;
  5106. spin_unlock(&block_rsv->lock);
  5107. return block_rsv;
  5108. } else if (ret && block_rsv != global_rsv) {
  5109. ret = block_rsv_use_bytes(global_rsv, blocksize);
  5110. if (!ret)
  5111. return global_rsv;
  5112. }
  5113. }
  5114. return ERR_PTR(-ENOSPC);
  5115. }
  5116. static void unuse_block_rsv(struct btrfs_block_rsv *block_rsv, u32 blocksize)
  5117. {
  5118. block_rsv_add_bytes(block_rsv, blocksize, 0);
  5119. block_rsv_release_bytes(block_rsv, NULL, 0);
  5120. }
  5121. /*
  5122. * finds a free extent and does all the dirty work required for allocation
  5123. * returns the key for the extent through ins, and a tree buffer for
  5124. * the first block of the extent through buf.
  5125. *
  5126. * returns the tree buffer or NULL.
  5127. */
  5128. struct extent_buffer *btrfs_alloc_free_block(struct btrfs_trans_handle *trans,
  5129. struct btrfs_root *root, u32 blocksize,
  5130. u64 parent, u64 root_objectid,
  5131. struct btrfs_disk_key *key, int level,
  5132. u64 hint, u64 empty_size)
  5133. {
  5134. struct btrfs_key ins;
  5135. struct btrfs_block_rsv *block_rsv;
  5136. struct extent_buffer *buf;
  5137. u64 flags = 0;
  5138. int ret;
  5139. block_rsv = use_block_rsv(trans, root, blocksize);
  5140. if (IS_ERR(block_rsv))
  5141. return ERR_CAST(block_rsv);
  5142. ret = btrfs_reserve_extent(trans, root, blocksize, blocksize,
  5143. empty_size, hint, (u64)-1, &ins, 0);
  5144. if (ret) {
  5145. unuse_block_rsv(block_rsv, blocksize);
  5146. return ERR_PTR(ret);
  5147. }
  5148. buf = btrfs_init_new_buffer(trans, root, ins.objectid,
  5149. blocksize, level);
  5150. BUG_ON(IS_ERR(buf));
  5151. if (root_objectid == BTRFS_TREE_RELOC_OBJECTID) {
  5152. if (parent == 0)
  5153. parent = ins.objectid;
  5154. flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF;
  5155. } else
  5156. BUG_ON(parent > 0);
  5157. if (root_objectid != BTRFS_TREE_LOG_OBJECTID) {
  5158. struct btrfs_delayed_extent_op *extent_op;
  5159. extent_op = kmalloc(sizeof(*extent_op), GFP_NOFS);
  5160. BUG_ON(!extent_op);
  5161. if (key)
  5162. memcpy(&extent_op->key, key, sizeof(extent_op->key));
  5163. else
  5164. memset(&extent_op->key, 0, sizeof(extent_op->key));
  5165. extent_op->flags_to_set = flags;
  5166. extent_op->update_key = 1;
  5167. extent_op->update_flags = 1;
  5168. extent_op->is_data = 0;
  5169. ret = btrfs_add_delayed_tree_ref(trans, ins.objectid,
  5170. ins.offset, parent, root_objectid,
  5171. level, BTRFS_ADD_DELAYED_EXTENT,
  5172. extent_op);
  5173. BUG_ON(ret);
  5174. }
  5175. return buf;
  5176. }
  5177. struct walk_control {
  5178. u64 refs[BTRFS_MAX_LEVEL];
  5179. u64 flags[BTRFS_MAX_LEVEL];
  5180. struct btrfs_key update_progress;
  5181. int stage;
  5182. int level;
  5183. int shared_level;
  5184. int update_ref;
  5185. int keep_locks;
  5186. int reada_slot;
  5187. int reada_count;
  5188. };
  5189. #define DROP_REFERENCE 1
  5190. #define UPDATE_BACKREF 2
  5191. static noinline void reada_walk_down(struct btrfs_trans_handle *trans,
  5192. struct btrfs_root *root,
  5193. struct walk_control *wc,
  5194. struct btrfs_path *path)
  5195. {
  5196. u64 bytenr;
  5197. u64 generation;
  5198. u64 refs;
  5199. u64 flags;
  5200. u32 nritems;
  5201. u32 blocksize;
  5202. struct btrfs_key key;
  5203. struct extent_buffer *eb;
  5204. int ret;
  5205. int slot;
  5206. int nread = 0;
  5207. if (path->slots[wc->level] < wc->reada_slot) {
  5208. wc->reada_count = wc->reada_count * 2 / 3;
  5209. wc->reada_count = max(wc->reada_count, 2);
  5210. } else {
  5211. wc->reada_count = wc->reada_count * 3 / 2;
  5212. wc->reada_count = min_t(int, wc->reada_count,
  5213. BTRFS_NODEPTRS_PER_BLOCK(root));
  5214. }
  5215. eb = path->nodes[wc->level];
  5216. nritems = btrfs_header_nritems(eb);
  5217. blocksize = btrfs_level_size(root, wc->level - 1);
  5218. for (slot = path->slots[wc->level]; slot < nritems; slot++) {
  5219. if (nread >= wc->reada_count)
  5220. break;
  5221. cond_resched();
  5222. bytenr = btrfs_node_blockptr(eb, slot);
  5223. generation = btrfs_node_ptr_generation(eb, slot);
  5224. if (slot == path->slots[wc->level])
  5225. goto reada;
  5226. if (wc->stage == UPDATE_BACKREF &&
  5227. generation <= root->root_key.offset)
  5228. continue;
  5229. /* We don't lock the tree block, it's OK to be racy here */
  5230. ret = btrfs_lookup_extent_info(trans, root, bytenr, blocksize,
  5231. &refs, &flags);
  5232. BUG_ON(ret);
  5233. BUG_ON(refs == 0);
  5234. if (wc->stage == DROP_REFERENCE) {
  5235. if (refs == 1)
  5236. goto reada;
  5237. if (wc->level == 1 &&
  5238. (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
  5239. continue;
  5240. if (!wc->update_ref ||
  5241. generation <= root->root_key.offset)
  5242. continue;
  5243. btrfs_node_key_to_cpu(eb, &key, slot);
  5244. ret = btrfs_comp_cpu_keys(&key,
  5245. &wc->update_progress);
  5246. if (ret < 0)
  5247. continue;
  5248. } else {
  5249. if (wc->level == 1 &&
  5250. (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
  5251. continue;
  5252. }
  5253. reada:
  5254. ret = readahead_tree_block(root, bytenr, blocksize,
  5255. generation);
  5256. if (ret)
  5257. break;
  5258. nread++;
  5259. }
  5260. wc->reada_slot = slot;
  5261. }
  5262. /*
  5263. * hepler to process tree block while walking down the tree.
  5264. *
  5265. * when wc->stage == UPDATE_BACKREF, this function updates
  5266. * back refs for pointers in the block.
  5267. *
  5268. * NOTE: return value 1 means we should stop walking down.
  5269. */
  5270. static noinline int walk_down_proc(struct btrfs_trans_handle *trans,
  5271. struct btrfs_root *root,
  5272. struct btrfs_path *path,
  5273. struct walk_control *wc, int lookup_info)
  5274. {
  5275. int level = wc->level;
  5276. struct extent_buffer *eb = path->nodes[level];
  5277. u64 flag = BTRFS_BLOCK_FLAG_FULL_BACKREF;
  5278. int ret;
  5279. if (wc->stage == UPDATE_BACKREF &&
  5280. btrfs_header_owner(eb) != root->root_key.objectid)
  5281. return 1;
  5282. /*
  5283. * when reference count of tree block is 1, it won't increase
  5284. * again. once full backref flag is set, we never clear it.
  5285. */
  5286. if (lookup_info &&
  5287. ((wc->stage == DROP_REFERENCE && wc->refs[level] != 1) ||
  5288. (wc->stage == UPDATE_BACKREF && !(wc->flags[level] & flag)))) {
  5289. BUG_ON(!path->locks[level]);
  5290. ret = btrfs_lookup_extent_info(trans, root,
  5291. eb->start, eb->len,
  5292. &wc->refs[level],
  5293. &wc->flags[level]);
  5294. BUG_ON(ret);
  5295. BUG_ON(wc->refs[level] == 0);
  5296. }
  5297. if (wc->stage == DROP_REFERENCE) {
  5298. if (wc->refs[level] > 1)
  5299. return 1;
  5300. if (path->locks[level] && !wc->keep_locks) {
  5301. btrfs_tree_unlock(eb);
  5302. path->locks[level] = 0;
  5303. }
  5304. return 0;
  5305. }
  5306. /* wc->stage == UPDATE_BACKREF */
  5307. if (!(wc->flags[level] & flag)) {
  5308. BUG_ON(!path->locks[level]);
  5309. ret = btrfs_inc_ref(trans, root, eb, 1);
  5310. BUG_ON(ret);
  5311. ret = btrfs_dec_ref(trans, root, eb, 0);
  5312. BUG_ON(ret);
  5313. ret = btrfs_set_disk_extent_flags(trans, root, eb->start,
  5314. eb->len, flag, 0);
  5315. BUG_ON(ret);
  5316. wc->flags[level] |= flag;
  5317. }
  5318. /*
  5319. * the block is shared by multiple trees, so it's not good to
  5320. * keep the tree lock
  5321. */
  5322. if (path->locks[level] && level > 0) {
  5323. btrfs_tree_unlock(eb);
  5324. path->locks[level] = 0;
  5325. }
  5326. return 0;
  5327. }
  5328. /*
  5329. * hepler to process tree block pointer.
  5330. *
  5331. * when wc->stage == DROP_REFERENCE, this function checks
  5332. * reference count of the block pointed to. if the block
  5333. * is shared and we need update back refs for the subtree
  5334. * rooted at the block, this function changes wc->stage to
  5335. * UPDATE_BACKREF. if the block is shared and there is no
  5336. * need to update back, this function drops the reference
  5337. * to the block.
  5338. *
  5339. * NOTE: return value 1 means we should stop walking down.
  5340. */
  5341. static noinline int do_walk_down(struct btrfs_trans_handle *trans,
  5342. struct btrfs_root *root,
  5343. struct btrfs_path *path,
  5344. struct walk_control *wc, int *lookup_info)
  5345. {
  5346. u64 bytenr;
  5347. u64 generation;
  5348. u64 parent;
  5349. u32 blocksize;
  5350. struct btrfs_key key;
  5351. struct extent_buffer *next;
  5352. int level = wc->level;
  5353. int reada = 0;
  5354. int ret = 0;
  5355. generation = btrfs_node_ptr_generation(path->nodes[level],
  5356. path->slots[level]);
  5357. /*
  5358. * if the lower level block was created before the snapshot
  5359. * was created, we know there is no need to update back refs
  5360. * for the subtree
  5361. */
  5362. if (wc->stage == UPDATE_BACKREF &&
  5363. generation <= root->root_key.offset) {
  5364. *lookup_info = 1;
  5365. return 1;
  5366. }
  5367. bytenr = btrfs_node_blockptr(path->nodes[level], path->slots[level]);
  5368. blocksize = btrfs_level_size(root, level - 1);
  5369. next = btrfs_find_tree_block(root, bytenr, blocksize);
  5370. if (!next) {
  5371. next = btrfs_find_create_tree_block(root, bytenr, blocksize);
  5372. if (!next)
  5373. return -ENOMEM;
  5374. reada = 1;
  5375. }
  5376. btrfs_tree_lock(next);
  5377. btrfs_set_lock_blocking(next);
  5378. ret = btrfs_lookup_extent_info(trans, root, bytenr, blocksize,
  5379. &wc->refs[level - 1],
  5380. &wc->flags[level - 1]);
  5381. BUG_ON(ret);
  5382. BUG_ON(wc->refs[level - 1] == 0);
  5383. *lookup_info = 0;
  5384. if (wc->stage == DROP_REFERENCE) {
  5385. if (wc->refs[level - 1] > 1) {
  5386. if (level == 1 &&
  5387. (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
  5388. goto skip;
  5389. if (!wc->update_ref ||
  5390. generation <= root->root_key.offset)
  5391. goto skip;
  5392. btrfs_node_key_to_cpu(path->nodes[level], &key,
  5393. path->slots[level]);
  5394. ret = btrfs_comp_cpu_keys(&key, &wc->update_progress);
  5395. if (ret < 0)
  5396. goto skip;
  5397. wc->stage = UPDATE_BACKREF;
  5398. wc->shared_level = level - 1;
  5399. }
  5400. } else {
  5401. if (level == 1 &&
  5402. (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
  5403. goto skip;
  5404. }
  5405. if (!btrfs_buffer_uptodate(next, generation)) {
  5406. btrfs_tree_unlock(next);
  5407. free_extent_buffer(next);
  5408. next = NULL;
  5409. *lookup_info = 1;
  5410. }
  5411. if (!next) {
  5412. if (reada && level == 1)
  5413. reada_walk_down(trans, root, wc, path);
  5414. next = read_tree_block(root, bytenr, blocksize, generation);
  5415. if (!next)
  5416. return -EIO;
  5417. btrfs_tree_lock(next);
  5418. btrfs_set_lock_blocking(next);
  5419. }
  5420. level--;
  5421. BUG_ON(level != btrfs_header_level(next));
  5422. path->nodes[level] = next;
  5423. path->slots[level] = 0;
  5424. path->locks[level] = 1;
  5425. wc->level = level;
  5426. if (wc->level == 1)
  5427. wc->reada_slot = 0;
  5428. return 0;
  5429. skip:
  5430. wc->refs[level - 1] = 0;
  5431. wc->flags[level - 1] = 0;
  5432. if (wc->stage == DROP_REFERENCE) {
  5433. if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
  5434. parent = path->nodes[level]->start;
  5435. } else {
  5436. BUG_ON(root->root_key.objectid !=
  5437. btrfs_header_owner(path->nodes[level]));
  5438. parent = 0;
  5439. }
  5440. ret = btrfs_free_extent(trans, root, bytenr, blocksize, parent,
  5441. root->root_key.objectid, level - 1, 0);
  5442. BUG_ON(ret);
  5443. }
  5444. btrfs_tree_unlock(next);
  5445. free_extent_buffer(next);
  5446. *lookup_info = 1;
  5447. return 1;
  5448. }
  5449. /*
  5450. * hepler to process tree block while walking up the tree.
  5451. *
  5452. * when wc->stage == DROP_REFERENCE, this function drops
  5453. * reference count on the block.
  5454. *
  5455. * when wc->stage == UPDATE_BACKREF, this function changes
  5456. * wc->stage back to DROP_REFERENCE if we changed wc->stage
  5457. * to UPDATE_BACKREF previously while processing the block.
  5458. *
  5459. * NOTE: return value 1 means we should stop walking up.
  5460. */
  5461. static noinline int walk_up_proc(struct btrfs_trans_handle *trans,
  5462. struct btrfs_root *root,
  5463. struct btrfs_path *path,
  5464. struct walk_control *wc)
  5465. {
  5466. int ret;
  5467. int level = wc->level;
  5468. struct extent_buffer *eb = path->nodes[level];
  5469. u64 parent = 0;
  5470. if (wc->stage == UPDATE_BACKREF) {
  5471. BUG_ON(wc->shared_level < level);
  5472. if (level < wc->shared_level)
  5473. goto out;
  5474. ret = find_next_key(path, level + 1, &wc->update_progress);
  5475. if (ret > 0)
  5476. wc->update_ref = 0;
  5477. wc->stage = DROP_REFERENCE;
  5478. wc->shared_level = -1;
  5479. path->slots[level] = 0;
  5480. /*
  5481. * check reference count again if the block isn't locked.
  5482. * we should start walking down the tree again if reference
  5483. * count is one.
  5484. */
  5485. if (!path->locks[level]) {
  5486. BUG_ON(level == 0);
  5487. btrfs_tree_lock(eb);
  5488. btrfs_set_lock_blocking(eb);
  5489. path->locks[level] = 1;
  5490. ret = btrfs_lookup_extent_info(trans, root,
  5491. eb->start, eb->len,
  5492. &wc->refs[level],
  5493. &wc->flags[level]);
  5494. BUG_ON(ret);
  5495. BUG_ON(wc->refs[level] == 0);
  5496. if (wc->refs[level] == 1) {
  5497. btrfs_tree_unlock(eb);
  5498. path->locks[level] = 0;
  5499. return 1;
  5500. }
  5501. }
  5502. }
  5503. /* wc->stage == DROP_REFERENCE */
  5504. BUG_ON(wc->refs[level] > 1 && !path->locks[level]);
  5505. if (wc->refs[level] == 1) {
  5506. if (level == 0) {
  5507. if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
  5508. ret = btrfs_dec_ref(trans, root, eb, 1);
  5509. else
  5510. ret = btrfs_dec_ref(trans, root, eb, 0);
  5511. BUG_ON(ret);
  5512. }
  5513. /* make block locked assertion in clean_tree_block happy */
  5514. if (!path->locks[level] &&
  5515. btrfs_header_generation(eb) == trans->transid) {
  5516. btrfs_tree_lock(eb);
  5517. btrfs_set_lock_blocking(eb);
  5518. path->locks[level] = 1;
  5519. }
  5520. clean_tree_block(trans, root, eb);
  5521. }
  5522. if (eb == root->node) {
  5523. if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
  5524. parent = eb->start;
  5525. else
  5526. BUG_ON(root->root_key.objectid !=
  5527. btrfs_header_owner(eb));
  5528. } else {
  5529. if (wc->flags[level + 1] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
  5530. parent = path->nodes[level + 1]->start;
  5531. else
  5532. BUG_ON(root->root_key.objectid !=
  5533. btrfs_header_owner(path->nodes[level + 1]));
  5534. }
  5535. btrfs_free_tree_block(trans, root, eb, parent, wc->refs[level] == 1);
  5536. out:
  5537. wc->refs[level] = 0;
  5538. wc->flags[level] = 0;
  5539. return 0;
  5540. }
  5541. static noinline int walk_down_tree(struct btrfs_trans_handle *trans,
  5542. struct btrfs_root *root,
  5543. struct btrfs_path *path,
  5544. struct walk_control *wc)
  5545. {
  5546. int level = wc->level;
  5547. int lookup_info = 1;
  5548. int ret;
  5549. while (level >= 0) {
  5550. ret = walk_down_proc(trans, root, path, wc, lookup_info);
  5551. if (ret > 0)
  5552. break;
  5553. if (level == 0)
  5554. break;
  5555. if (path->slots[level] >=
  5556. btrfs_header_nritems(path->nodes[level]))
  5557. break;
  5558. ret = do_walk_down(trans, root, path, wc, &lookup_info);
  5559. if (ret > 0) {
  5560. path->slots[level]++;
  5561. continue;
  5562. } else if (ret < 0)
  5563. return ret;
  5564. level = wc->level;
  5565. }
  5566. return 0;
  5567. }
  5568. static noinline int walk_up_tree(struct btrfs_trans_handle *trans,
  5569. struct btrfs_root *root,
  5570. struct btrfs_path *path,
  5571. struct walk_control *wc, int max_level)
  5572. {
  5573. int level = wc->level;
  5574. int ret;
  5575. path->slots[level] = btrfs_header_nritems(path->nodes[level]);
  5576. while (level < max_level && path->nodes[level]) {
  5577. wc->level = level;
  5578. if (path->slots[level] + 1 <
  5579. btrfs_header_nritems(path->nodes[level])) {
  5580. path->slots[level]++;
  5581. return 0;
  5582. } else {
  5583. ret = walk_up_proc(trans, root, path, wc);
  5584. if (ret > 0)
  5585. return 0;
  5586. if (path->locks[level]) {
  5587. btrfs_tree_unlock(path->nodes[level]);
  5588. path->locks[level] = 0;
  5589. }
  5590. free_extent_buffer(path->nodes[level]);
  5591. path->nodes[level] = NULL;
  5592. level++;
  5593. }
  5594. }
  5595. return 1;
  5596. }
  5597. /*
  5598. * drop a subvolume tree.
  5599. *
  5600. * this function traverses the tree freeing any blocks that only
  5601. * referenced by the tree.
  5602. *
  5603. * when a shared tree block is found. this function decreases its
  5604. * reference count by one. if update_ref is true, this function
  5605. * also make sure backrefs for the shared block and all lower level
  5606. * blocks are properly updated.
  5607. */
  5608. int btrfs_drop_snapshot(struct btrfs_root *root,
  5609. struct btrfs_block_rsv *block_rsv, int update_ref)
  5610. {
  5611. struct btrfs_path *path;
  5612. struct btrfs_trans_handle *trans;
  5613. struct btrfs_root *tree_root = root->fs_info->tree_root;
  5614. struct btrfs_root_item *root_item = &root->root_item;
  5615. struct walk_control *wc;
  5616. struct btrfs_key key;
  5617. int err = 0;
  5618. int ret;
  5619. int level;
  5620. path = btrfs_alloc_path();
  5621. BUG_ON(!path);
  5622. wc = kzalloc(sizeof(*wc), GFP_NOFS);
  5623. BUG_ON(!wc);
  5624. trans = btrfs_start_transaction(tree_root, 0);
  5625. BUG_ON(IS_ERR(trans));
  5626. if (block_rsv)
  5627. trans->block_rsv = block_rsv;
  5628. if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) {
  5629. level = btrfs_header_level(root->node);
  5630. path->nodes[level] = btrfs_lock_root_node(root);
  5631. btrfs_set_lock_blocking(path->nodes[level]);
  5632. path->slots[level] = 0;
  5633. path->locks[level] = 1;
  5634. memset(&wc->update_progress, 0,
  5635. sizeof(wc->update_progress));
  5636. } else {
  5637. btrfs_disk_key_to_cpu(&key, &root_item->drop_progress);
  5638. memcpy(&wc->update_progress, &key,
  5639. sizeof(wc->update_progress));
  5640. level = root_item->drop_level;
  5641. BUG_ON(level == 0);
  5642. path->lowest_level = level;
  5643. ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
  5644. path->lowest_level = 0;
  5645. if (ret < 0) {
  5646. err = ret;
  5647. goto out;
  5648. }
  5649. WARN_ON(ret > 0);
  5650. /*
  5651. * unlock our path, this is safe because only this
  5652. * function is allowed to delete this snapshot
  5653. */
  5654. btrfs_unlock_up_safe(path, 0);
  5655. level = btrfs_header_level(root->node);
  5656. while (1) {
  5657. btrfs_tree_lock(path->nodes[level]);
  5658. btrfs_set_lock_blocking(path->nodes[level]);
  5659. ret = btrfs_lookup_extent_info(trans, root,
  5660. path->nodes[level]->start,
  5661. path->nodes[level]->len,
  5662. &wc->refs[level],
  5663. &wc->flags[level]);
  5664. BUG_ON(ret);
  5665. BUG_ON(wc->refs[level] == 0);
  5666. if (level == root_item->drop_level)
  5667. break;
  5668. btrfs_tree_unlock(path->nodes[level]);
  5669. WARN_ON(wc->refs[level] != 1);
  5670. level--;
  5671. }
  5672. }
  5673. wc->level = level;
  5674. wc->shared_level = -1;
  5675. wc->stage = DROP_REFERENCE;
  5676. wc->update_ref = update_ref;
  5677. wc->keep_locks = 0;
  5678. wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
  5679. while (1) {
  5680. ret = walk_down_tree(trans, root, path, wc);
  5681. if (ret < 0) {
  5682. err = ret;
  5683. break;
  5684. }
  5685. ret = walk_up_tree(trans, root, path, wc, BTRFS_MAX_LEVEL);
  5686. if (ret < 0) {
  5687. err = ret;
  5688. break;
  5689. }
  5690. if (ret > 0) {
  5691. BUG_ON(wc->stage != DROP_REFERENCE);
  5692. break;
  5693. }
  5694. if (wc->stage == DROP_REFERENCE) {
  5695. level = wc->level;
  5696. btrfs_node_key(path->nodes[level],
  5697. &root_item->drop_progress,
  5698. path->slots[level]);
  5699. root_item->drop_level = level;
  5700. }
  5701. BUG_ON(wc->level == 0);
  5702. if (btrfs_should_end_transaction(trans, tree_root)) {
  5703. ret = btrfs_update_root(trans, tree_root,
  5704. &root->root_key,
  5705. root_item);
  5706. BUG_ON(ret);
  5707. btrfs_end_transaction_throttle(trans, tree_root);
  5708. trans = btrfs_start_transaction(tree_root, 0);
  5709. BUG_ON(IS_ERR(trans));
  5710. if (block_rsv)
  5711. trans->block_rsv = block_rsv;
  5712. }
  5713. }
  5714. btrfs_release_path(root, path);
  5715. BUG_ON(err);
  5716. ret = btrfs_del_root(trans, tree_root, &root->root_key);
  5717. BUG_ON(ret);
  5718. if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) {
  5719. ret = btrfs_find_last_root(tree_root, root->root_key.objectid,
  5720. NULL, NULL);
  5721. BUG_ON(ret < 0);
  5722. if (ret > 0) {
  5723. /* if we fail to delete the orphan item this time
  5724. * around, it'll get picked up the next time.
  5725. *
  5726. * The most common failure here is just -ENOENT.
  5727. */
  5728. btrfs_del_orphan_item(trans, tree_root,
  5729. root->root_key.objectid);
  5730. }
  5731. }
  5732. if (root->in_radix) {
  5733. btrfs_free_fs_root(tree_root->fs_info, root);
  5734. } else {
  5735. free_extent_buffer(root->node);
  5736. free_extent_buffer(root->commit_root);
  5737. kfree(root);
  5738. }
  5739. out:
  5740. btrfs_end_transaction_throttle(trans, tree_root);
  5741. kfree(wc);
  5742. btrfs_free_path(path);
  5743. return err;
  5744. }
  5745. /*
  5746. * drop subtree rooted at tree block 'node'.
  5747. *
  5748. * NOTE: this function will unlock and release tree block 'node'
  5749. */
  5750. int btrfs_drop_subtree(struct btrfs_trans_handle *trans,
  5751. struct btrfs_root *root,
  5752. struct extent_buffer *node,
  5753. struct extent_buffer *parent)
  5754. {
  5755. struct btrfs_path *path;
  5756. struct walk_control *wc;
  5757. int level;
  5758. int parent_level;
  5759. int ret = 0;
  5760. int wret;
  5761. BUG_ON(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID);
  5762. path = btrfs_alloc_path();
  5763. if (!path)
  5764. return -ENOMEM;
  5765. wc = kzalloc(sizeof(*wc), GFP_NOFS);
  5766. if (!wc) {
  5767. btrfs_free_path(path);
  5768. return -ENOMEM;
  5769. }
  5770. btrfs_assert_tree_locked(parent);
  5771. parent_level = btrfs_header_level(parent);
  5772. extent_buffer_get(parent);
  5773. path->nodes[parent_level] = parent;
  5774. path->slots[parent_level] = btrfs_header_nritems(parent);
  5775. btrfs_assert_tree_locked(node);
  5776. level = btrfs_header_level(node);
  5777. path->nodes[level] = node;
  5778. path->slots[level] = 0;
  5779. path->locks[level] = 1;
  5780. wc->refs[parent_level] = 1;
  5781. wc->flags[parent_level] = BTRFS_BLOCK_FLAG_FULL_BACKREF;
  5782. wc->level = level;
  5783. wc->shared_level = -1;
  5784. wc->stage = DROP_REFERENCE;
  5785. wc->update_ref = 0;
  5786. wc->keep_locks = 1;
  5787. wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
  5788. while (1) {
  5789. wret = walk_down_tree(trans, root, path, wc);
  5790. if (wret < 0) {
  5791. ret = wret;
  5792. break;
  5793. }
  5794. wret = walk_up_tree(trans, root, path, wc, parent_level);
  5795. if (wret < 0)
  5796. ret = wret;
  5797. if (wret != 0)
  5798. break;
  5799. }
  5800. kfree(wc);
  5801. btrfs_free_path(path);
  5802. return ret;
  5803. }
  5804. #if 0
  5805. static unsigned long calc_ra(unsigned long start, unsigned long last,
  5806. unsigned long nr)
  5807. {
  5808. return min(last, start + nr - 1);
  5809. }
  5810. static noinline int relocate_inode_pages(struct inode *inode, u64 start,
  5811. u64 len)
  5812. {
  5813. u64 page_start;
  5814. u64 page_end;
  5815. unsigned long first_index;
  5816. unsigned long last_index;
  5817. unsigned long i;
  5818. struct page *page;
  5819. struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
  5820. struct file_ra_state *ra;
  5821. struct btrfs_ordered_extent *ordered;
  5822. unsigned int total_read = 0;
  5823. unsigned int total_dirty = 0;
  5824. int ret = 0;
  5825. ra = kzalloc(sizeof(*ra), GFP_NOFS);
  5826. if (!ra)
  5827. return -ENOMEM;
  5828. mutex_lock(&inode->i_mutex);
  5829. first_index = start >> PAGE_CACHE_SHIFT;
  5830. last_index = (start + len - 1) >> PAGE_CACHE_SHIFT;
  5831. /* make sure the dirty trick played by the caller work */
  5832. ret = invalidate_inode_pages2_range(inode->i_mapping,
  5833. first_index, last_index);
  5834. if (ret)
  5835. goto out_unlock;
  5836. file_ra_state_init(ra, inode->i_mapping);
  5837. for (i = first_index ; i <= last_index; i++) {
  5838. if (total_read % ra->ra_pages == 0) {
  5839. btrfs_force_ra(inode->i_mapping, ra, NULL, i,
  5840. calc_ra(i, last_index, ra->ra_pages));
  5841. }
  5842. total_read++;
  5843. again:
  5844. if (((u64)i << PAGE_CACHE_SHIFT) > i_size_read(inode))
  5845. BUG_ON(1);
  5846. page = grab_cache_page(inode->i_mapping, i);
  5847. if (!page) {
  5848. ret = -ENOMEM;
  5849. goto out_unlock;
  5850. }
  5851. if (!PageUptodate(page)) {
  5852. btrfs_readpage(NULL, page);
  5853. lock_page(page);
  5854. if (!PageUptodate(page)) {
  5855. unlock_page(page);
  5856. page_cache_release(page);
  5857. ret = -EIO;
  5858. goto out_unlock;
  5859. }
  5860. }
  5861. wait_on_page_writeback(page);
  5862. page_start = (u64)page->index << PAGE_CACHE_SHIFT;
  5863. page_end = page_start + PAGE_CACHE_SIZE - 1;
  5864. lock_extent(io_tree, page_start, page_end, GFP_NOFS);
  5865. ordered = btrfs_lookup_ordered_extent(inode, page_start);
  5866. if (ordered) {
  5867. unlock_extent(io_tree, page_start, page_end, GFP_NOFS);
  5868. unlock_page(page);
  5869. page_cache_release(page);
  5870. btrfs_start_ordered_extent(inode, ordered, 1);
  5871. btrfs_put_ordered_extent(ordered);
  5872. goto again;
  5873. }
  5874. set_page_extent_mapped(page);
  5875. if (i == first_index)
  5876. set_extent_bits(io_tree, page_start, page_end,
  5877. EXTENT_BOUNDARY, GFP_NOFS);
  5878. btrfs_set_extent_delalloc(inode, page_start, page_end);
  5879. set_page_dirty(page);
  5880. total_dirty++;
  5881. unlock_extent(io_tree, page_start, page_end, GFP_NOFS);
  5882. unlock_page(page);
  5883. page_cache_release(page);
  5884. }
  5885. out_unlock:
  5886. kfree(ra);
  5887. mutex_unlock(&inode->i_mutex);
  5888. balance_dirty_pages_ratelimited_nr(inode->i_mapping, total_dirty);
  5889. return ret;
  5890. }
  5891. static noinline int relocate_data_extent(struct inode *reloc_inode,
  5892. struct btrfs_key *extent_key,
  5893. u64 offset)
  5894. {
  5895. struct btrfs_root *root = BTRFS_I(reloc_inode)->root;
  5896. struct extent_map_tree *em_tree = &BTRFS_I(reloc_inode)->extent_tree;
  5897. struct extent_map *em;
  5898. u64 start = extent_key->objectid - offset;
  5899. u64 end = start + extent_key->offset - 1;
  5900. em = alloc_extent_map(GFP_NOFS);
  5901. BUG_ON(!em);
  5902. em->start = start;
  5903. em->len = extent_key->offset;
  5904. em->block_len = extent_key->offset;
  5905. em->block_start = extent_key->objectid;
  5906. em->bdev = root->fs_info->fs_devices->latest_bdev;
  5907. set_bit(EXTENT_FLAG_PINNED, &em->flags);
  5908. /* setup extent map to cheat btrfs_readpage */
  5909. lock_extent(&BTRFS_I(reloc_inode)->io_tree, start, end, GFP_NOFS);
  5910. while (1) {
  5911. int ret;
  5912. write_lock(&em_tree->lock);
  5913. ret = add_extent_mapping(em_tree, em);
  5914. write_unlock(&em_tree->lock);
  5915. if (ret != -EEXIST) {
  5916. free_extent_map(em);
  5917. break;
  5918. }
  5919. btrfs_drop_extent_cache(reloc_inode, start, end, 0);
  5920. }
  5921. unlock_extent(&BTRFS_I(reloc_inode)->io_tree, start, end, GFP_NOFS);
  5922. return relocate_inode_pages(reloc_inode, start, extent_key->offset);
  5923. }
  5924. struct btrfs_ref_path {
  5925. u64 extent_start;
  5926. u64 nodes[BTRFS_MAX_LEVEL];
  5927. u64 root_objectid;
  5928. u64 root_generation;
  5929. u64 owner_objectid;
  5930. u32 num_refs;
  5931. int lowest_level;
  5932. int current_level;
  5933. int shared_level;
  5934. struct btrfs_key node_keys[BTRFS_MAX_LEVEL];
  5935. u64 new_nodes[BTRFS_MAX_LEVEL];
  5936. };
  5937. struct disk_extent {
  5938. u64 ram_bytes;
  5939. u64 disk_bytenr;
  5940. u64 disk_num_bytes;
  5941. u64 offset;
  5942. u64 num_bytes;
  5943. u8 compression;
  5944. u8 encryption;
  5945. u16 other_encoding;
  5946. };
  5947. static int is_cowonly_root(u64 root_objectid)
  5948. {
  5949. if (root_objectid == BTRFS_ROOT_TREE_OBJECTID ||
  5950. root_objectid == BTRFS_EXTENT_TREE_OBJECTID ||
  5951. root_objectid == BTRFS_CHUNK_TREE_OBJECTID ||
  5952. root_objectid == BTRFS_DEV_TREE_OBJECTID ||
  5953. root_objectid == BTRFS_TREE_LOG_OBJECTID ||
  5954. root_objectid == BTRFS_CSUM_TREE_OBJECTID)
  5955. return 1;
  5956. return 0;
  5957. }
  5958. static noinline int __next_ref_path(struct btrfs_trans_handle *trans,
  5959. struct btrfs_root *extent_root,
  5960. struct btrfs_ref_path *ref_path,
  5961. int first_time)
  5962. {
  5963. struct extent_buffer *leaf;
  5964. struct btrfs_path *path;
  5965. struct btrfs_extent_ref *ref;
  5966. struct btrfs_key key;
  5967. struct btrfs_key found_key;
  5968. u64 bytenr;
  5969. u32 nritems;
  5970. int level;
  5971. int ret = 1;
  5972. path = btrfs_alloc_path();
  5973. if (!path)
  5974. return -ENOMEM;
  5975. if (first_time) {
  5976. ref_path->lowest_level = -1;
  5977. ref_path->current_level = -1;
  5978. ref_path->shared_level = -1;
  5979. goto walk_up;
  5980. }
  5981. walk_down:
  5982. level = ref_path->current_level - 1;
  5983. while (level >= -1) {
  5984. u64 parent;
  5985. if (level < ref_path->lowest_level)
  5986. break;
  5987. if (level >= 0)
  5988. bytenr = ref_path->nodes[level];
  5989. else
  5990. bytenr = ref_path->extent_start;
  5991. BUG_ON(bytenr == 0);
  5992. parent = ref_path->nodes[level + 1];
  5993. ref_path->nodes[level + 1] = 0;
  5994. ref_path->current_level = level;
  5995. BUG_ON(parent == 0);
  5996. key.objectid = bytenr;
  5997. key.offset = parent + 1;
  5998. key.type = BTRFS_EXTENT_REF_KEY;
  5999. ret = btrfs_search_slot(trans, extent_root, &key, path, 0, 0);
  6000. if (ret < 0)
  6001. goto out;
  6002. BUG_ON(ret == 0);
  6003. leaf = path->nodes[0];
  6004. nritems = btrfs_header_nritems(leaf);
  6005. if (path->slots[0] >= nritems) {
  6006. ret = btrfs_next_leaf(extent_root, path);
  6007. if (ret < 0)
  6008. goto out;
  6009. if (ret > 0)
  6010. goto next;
  6011. leaf = path->nodes[0];
  6012. }
  6013. btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
  6014. if (found_key.objectid == bytenr &&
  6015. found_key.type == BTRFS_EXTENT_REF_KEY) {
  6016. if (level < ref_path->shared_level)
  6017. ref_path->shared_level = level;
  6018. goto found;
  6019. }
  6020. next:
  6021. level--;
  6022. btrfs_release_path(extent_root, path);
  6023. cond_resched();
  6024. }
  6025. /* reached lowest level */
  6026. ret = 1;
  6027. goto out;
  6028. walk_up:
  6029. level = ref_path->current_level;
  6030. while (level < BTRFS_MAX_LEVEL - 1) {
  6031. u64 ref_objectid;
  6032. if (level >= 0)
  6033. bytenr = ref_path->nodes[level];
  6034. else
  6035. bytenr = ref_path->extent_start;
  6036. BUG_ON(bytenr == 0);
  6037. key.objectid = bytenr;
  6038. key.offset = 0;
  6039. key.type = BTRFS_EXTENT_REF_KEY;
  6040. ret = btrfs_search_slot(trans, extent_root, &key, path, 0, 0);
  6041. if (ret < 0)
  6042. goto out;
  6043. leaf = path->nodes[0];
  6044. nritems = btrfs_header_nritems(leaf);
  6045. if (path->slots[0] >= nritems) {
  6046. ret = btrfs_next_leaf(extent_root, path);
  6047. if (ret < 0)
  6048. goto out;
  6049. if (ret > 0) {
  6050. /* the extent was freed by someone */
  6051. if (ref_path->lowest_level == level)
  6052. goto out;
  6053. btrfs_release_path(extent_root, path);
  6054. goto walk_down;
  6055. }
  6056. leaf = path->nodes[0];
  6057. }
  6058. btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
  6059. if (found_key.objectid != bytenr ||
  6060. found_key.type != BTRFS_EXTENT_REF_KEY) {
  6061. /* the extent was freed by someone */
  6062. if (ref_path->lowest_level == level) {
  6063. ret = 1;
  6064. goto out;
  6065. }
  6066. btrfs_release_path(extent_root, path);
  6067. goto walk_down;
  6068. }
  6069. found:
  6070. ref = btrfs_item_ptr(leaf, path->slots[0],
  6071. struct btrfs_extent_ref);
  6072. ref_objectid = btrfs_ref_objectid(leaf, ref);
  6073. if (ref_objectid < BTRFS_FIRST_FREE_OBJECTID) {
  6074. if (first_time) {
  6075. level = (int)ref_objectid;
  6076. BUG_ON(level >= BTRFS_MAX_LEVEL);
  6077. ref_path->lowest_level = level;
  6078. ref_path->current_level = level;
  6079. ref_path->nodes[level] = bytenr;
  6080. } else {
  6081. WARN_ON(ref_objectid != level);
  6082. }
  6083. } else {
  6084. WARN_ON(level != -1);
  6085. }
  6086. first_time = 0;
  6087. if (ref_path->lowest_level == level) {
  6088. ref_path->owner_objectid = ref_objectid;
  6089. ref_path->num_refs = btrfs_ref_num_refs(leaf, ref);
  6090. }
  6091. /*
  6092. * the block is tree root or the block isn't in reference
  6093. * counted tree.
  6094. */
  6095. if (found_key.objectid == found_key.offset ||
  6096. is_cowonly_root(btrfs_ref_root(leaf, ref))) {
  6097. ref_path->root_objectid = btrfs_ref_root(leaf, ref);
  6098. ref_path->root_generation =
  6099. btrfs_ref_generation(leaf, ref);
  6100. if (level < 0) {
  6101. /* special reference from the tree log */
  6102. ref_path->nodes[0] = found_key.offset;
  6103. ref_path->current_level = 0;
  6104. }
  6105. ret = 0;
  6106. goto out;
  6107. }
  6108. level++;
  6109. BUG_ON(ref_path->nodes[level] != 0);
  6110. ref_path->nodes[level] = found_key.offset;
  6111. ref_path->current_level = level;
  6112. /*
  6113. * the reference was created in the running transaction,
  6114. * no need to continue walking up.
  6115. */
  6116. if (btrfs_ref_generation(leaf, ref) == trans->transid) {
  6117. ref_path->root_objectid = btrfs_ref_root(leaf, ref);
  6118. ref_path->root_generation =
  6119. btrfs_ref_generation(leaf, ref);
  6120. ret = 0;
  6121. goto out;
  6122. }
  6123. btrfs_release_path(extent_root, path);
  6124. cond_resched();
  6125. }
  6126. /* reached max tree level, but no tree root found. */
  6127. BUG();
  6128. out:
  6129. btrfs_free_path(path);
  6130. return ret;
  6131. }
  6132. static int btrfs_first_ref_path(struct btrfs_trans_handle *trans,
  6133. struct btrfs_root *extent_root,
  6134. struct btrfs_ref_path *ref_path,
  6135. u64 extent_start)
  6136. {
  6137. memset(ref_path, 0, sizeof(*ref_path));
  6138. ref_path->extent_start = extent_start;
  6139. return __next_ref_path(trans, extent_root, ref_path, 1);
  6140. }
  6141. static int btrfs_next_ref_path(struct btrfs_trans_handle *trans,
  6142. struct btrfs_root *extent_root,
  6143. struct btrfs_ref_path *ref_path)
  6144. {
  6145. return __next_ref_path(trans, extent_root, ref_path, 0);
  6146. }
  6147. static noinline int get_new_locations(struct inode *reloc_inode,
  6148. struct btrfs_key *extent_key,
  6149. u64 offset, int no_fragment,
  6150. struct disk_extent **extents,
  6151. int *nr_extents)
  6152. {
  6153. struct btrfs_root *root = BTRFS_I(reloc_inode)->root;
  6154. struct btrfs_path *path;
  6155. struct btrfs_file_extent_item *fi;
  6156. struct extent_buffer *leaf;
  6157. struct disk_extent *exts = *extents;
  6158. struct btrfs_key found_key;
  6159. u64 cur_pos;
  6160. u64 last_byte;
  6161. u32 nritems;
  6162. int nr = 0;
  6163. int max = *nr_extents;
  6164. int ret;
  6165. WARN_ON(!no_fragment && *extents);
  6166. if (!exts) {
  6167. max = 1;
  6168. exts = kmalloc(sizeof(*exts) * max, GFP_NOFS);
  6169. if (!exts)
  6170. return -ENOMEM;
  6171. }
  6172. path = btrfs_alloc_path();
  6173. if (!path) {
  6174. if (exts != *extents)
  6175. kfree(exts);
  6176. return -ENOMEM;
  6177. }
  6178. cur_pos = extent_key->objectid - offset;
  6179. last_byte = extent_key->objectid + extent_key->offset;
  6180. ret = btrfs_lookup_file_extent(NULL, root, path,
  6181. btrfs_ino(reloc_inode), cur_pos, 0);
  6182. if (ret < 0)
  6183. goto out;
  6184. if (ret > 0) {
  6185. ret = -ENOENT;
  6186. goto out;
  6187. }
  6188. while (1) {
  6189. leaf = path->nodes[0];
  6190. nritems = btrfs_header_nritems(leaf);
  6191. if (path->slots[0] >= nritems) {
  6192. ret = btrfs_next_leaf(root, path);
  6193. if (ret < 0)
  6194. goto out;
  6195. if (ret > 0)
  6196. break;
  6197. leaf = path->nodes[0];
  6198. }
  6199. btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
  6200. if (found_key.offset != cur_pos ||
  6201. found_key.type != BTRFS_EXTENT_DATA_KEY ||
  6202. found_key.objectid != btrfs_ino(reloc_inode))
  6203. break;
  6204. fi = btrfs_item_ptr(leaf, path->slots[0],
  6205. struct btrfs_file_extent_item);
  6206. if (btrfs_file_extent_type(leaf, fi) !=
  6207. BTRFS_FILE_EXTENT_REG ||
  6208. btrfs_file_extent_disk_bytenr(leaf, fi) == 0)
  6209. break;
  6210. if (nr == max) {
  6211. struct disk_extent *old = exts;
  6212. max *= 2;
  6213. exts = kzalloc(sizeof(*exts) * max, GFP_NOFS);
  6214. if (!exts) {
  6215. ret = -ENOMEM;
  6216. goto out;
  6217. }
  6218. memcpy(exts, old, sizeof(*exts) * nr);
  6219. if (old != *extents)
  6220. kfree(old);
  6221. }
  6222. exts[nr].disk_bytenr =
  6223. btrfs_file_extent_disk_bytenr(leaf, fi);
  6224. exts[nr].disk_num_bytes =
  6225. btrfs_file_extent_disk_num_bytes(leaf, fi);
  6226. exts[nr].offset = btrfs_file_extent_offset(leaf, fi);
  6227. exts[nr].num_bytes = btrfs_file_extent_num_bytes(leaf, fi);
  6228. exts[nr].ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
  6229. exts[nr].compression = btrfs_file_extent_compression(leaf, fi);
  6230. exts[nr].encryption = btrfs_file_extent_encryption(leaf, fi);
  6231. exts[nr].other_encoding = btrfs_file_extent_other_encoding(leaf,
  6232. fi);
  6233. BUG_ON(exts[nr].offset > 0);
  6234. BUG_ON(exts[nr].compression || exts[nr].encryption);
  6235. BUG_ON(exts[nr].num_bytes != exts[nr].disk_num_bytes);
  6236. cur_pos += exts[nr].num_bytes;
  6237. nr++;
  6238. if (cur_pos + offset >= last_byte)
  6239. break;
  6240. if (no_fragment) {
  6241. ret = 1;
  6242. goto out;
  6243. }
  6244. path->slots[0]++;
  6245. }
  6246. BUG_ON(cur_pos + offset > last_byte);
  6247. if (cur_pos + offset < last_byte) {
  6248. ret = -ENOENT;
  6249. goto out;
  6250. }
  6251. ret = 0;
  6252. out:
  6253. btrfs_free_path(path);
  6254. if (ret) {
  6255. if (exts != *extents)
  6256. kfree(exts);
  6257. } else {
  6258. *extents = exts;
  6259. *nr_extents = nr;
  6260. }
  6261. return ret;
  6262. }
  6263. static noinline int replace_one_extent(struct btrfs_trans_handle *trans,
  6264. struct btrfs_root *root,
  6265. struct btrfs_path *path,
  6266. struct btrfs_key *extent_key,
  6267. struct btrfs_key *leaf_key,
  6268. struct btrfs_ref_path *ref_path,
  6269. struct disk_extent *new_extents,
  6270. int nr_extents)
  6271. {
  6272. struct extent_buffer *leaf;
  6273. struct btrfs_file_extent_item *fi;
  6274. struct inode *inode = NULL;
  6275. struct btrfs_key key;
  6276. u64 lock_start = 0;
  6277. u64 lock_end = 0;
  6278. u64 num_bytes;
  6279. u64 ext_offset;
  6280. u64 search_end = (u64)-1;
  6281. u32 nritems;
  6282. int nr_scaned = 0;
  6283. int extent_locked = 0;
  6284. int extent_type;
  6285. int ret;
  6286. memcpy(&key, leaf_key, sizeof(key));
  6287. if (ref_path->owner_objectid != BTRFS_MULTIPLE_OBJECTIDS) {
  6288. if (key.objectid < ref_path->owner_objectid ||
  6289. (key.objectid == ref_path->owner_objectid &&
  6290. key.type < BTRFS_EXTENT_DATA_KEY)) {
  6291. key.objectid = ref_path->owner_objectid;
  6292. key.type = BTRFS_EXTENT_DATA_KEY;
  6293. key.offset = 0;
  6294. }
  6295. }
  6296. while (1) {
  6297. ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
  6298. if (ret < 0)
  6299. goto out;
  6300. leaf = path->nodes[0];
  6301. nritems = btrfs_header_nritems(leaf);
  6302. next:
  6303. if (extent_locked && ret > 0) {
  6304. /*
  6305. * the file extent item was modified by someone
  6306. * before the extent got locked.
  6307. */
  6308. unlock_extent(&BTRFS_I(inode)->io_tree, lock_start,
  6309. lock_end, GFP_NOFS);
  6310. extent_locked = 0;
  6311. }
  6312. if (path->slots[0] >= nritems) {
  6313. if (++nr_scaned > 2)
  6314. break;
  6315. BUG_ON(extent_locked);
  6316. ret = btrfs_next_leaf(root, path);
  6317. if (ret < 0)
  6318. goto out;
  6319. if (ret > 0)
  6320. break;
  6321. leaf = path->nodes[0];
  6322. nritems = btrfs_header_nritems(leaf);
  6323. }
  6324. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  6325. if (ref_path->owner_objectid != BTRFS_MULTIPLE_OBJECTIDS) {
  6326. if ((key.objectid > ref_path->owner_objectid) ||
  6327. (key.objectid == ref_path->owner_objectid &&
  6328. key.type > BTRFS_EXTENT_DATA_KEY) ||
  6329. key.offset >= search_end)
  6330. break;
  6331. }
  6332. if (inode && key.objectid != btrfs_ino(inode)) {
  6333. BUG_ON(extent_locked);
  6334. btrfs_release_path(root, path);
  6335. mutex_unlock(&inode->i_mutex);
  6336. iput(inode);
  6337. inode = NULL;
  6338. continue;
  6339. }
  6340. if (key.type != BTRFS_EXTENT_DATA_KEY) {
  6341. path->slots[0]++;
  6342. ret = 1;
  6343. goto next;
  6344. }
  6345. fi = btrfs_item_ptr(leaf, path->slots[0],
  6346. struct btrfs_file_extent_item);
  6347. extent_type = btrfs_file_extent_type(leaf, fi);
  6348. if ((extent_type != BTRFS_FILE_EXTENT_REG &&
  6349. extent_type != BTRFS_FILE_EXTENT_PREALLOC) ||
  6350. (btrfs_file_extent_disk_bytenr(leaf, fi) !=
  6351. extent_key->objectid)) {
  6352. path->slots[0]++;
  6353. ret = 1;
  6354. goto next;
  6355. }
  6356. num_bytes = btrfs_file_extent_num_bytes(leaf, fi);
  6357. ext_offset = btrfs_file_extent_offset(leaf, fi);
  6358. if (search_end == (u64)-1) {
  6359. search_end = key.offset - ext_offset +
  6360. btrfs_file_extent_ram_bytes(leaf, fi);
  6361. }
  6362. if (!extent_locked) {
  6363. lock_start = key.offset;
  6364. lock_end = lock_start + num_bytes - 1;
  6365. } else {
  6366. if (lock_start > key.offset ||
  6367. lock_end + 1 < key.offset + num_bytes) {
  6368. unlock_extent(&BTRFS_I(inode)->io_tree,
  6369. lock_start, lock_end, GFP_NOFS);
  6370. extent_locked = 0;
  6371. }
  6372. }
  6373. if (!inode) {
  6374. btrfs_release_path(root, path);
  6375. inode = btrfs_iget_locked(root->fs_info->sb,
  6376. key.objectid, root);
  6377. if (inode->i_state & I_NEW) {
  6378. BTRFS_I(inode)->root = root;
  6379. BTRFS_I(inode)->location.objectid =
  6380. key.objectid;
  6381. BTRFS_I(inode)->location.type =
  6382. BTRFS_INODE_ITEM_KEY;
  6383. BTRFS_I(inode)->location.offset = 0;
  6384. btrfs_read_locked_inode(inode);
  6385. unlock_new_inode(inode);
  6386. }
  6387. /*
  6388. * some code call btrfs_commit_transaction while
  6389. * holding the i_mutex, so we can't use mutex_lock
  6390. * here.
  6391. */
  6392. if (is_bad_inode(inode) ||
  6393. !mutex_trylock(&inode->i_mutex)) {
  6394. iput(inode);
  6395. inode = NULL;
  6396. key.offset = (u64)-1;
  6397. goto skip;
  6398. }
  6399. }
  6400. if (!extent_locked) {
  6401. struct btrfs_ordered_extent *ordered;
  6402. btrfs_release_path(root, path);
  6403. lock_extent(&BTRFS_I(inode)->io_tree, lock_start,
  6404. lock_end, GFP_NOFS);
  6405. ordered = btrfs_lookup_first_ordered_extent(inode,
  6406. lock_end);
  6407. if (ordered &&
  6408. ordered->file_offset <= lock_end &&
  6409. ordered->file_offset + ordered->len > lock_start) {
  6410. unlock_extent(&BTRFS_I(inode)->io_tree,
  6411. lock_start, lock_end, GFP_NOFS);
  6412. btrfs_start_ordered_extent(inode, ordered, 1);
  6413. btrfs_put_ordered_extent(ordered);
  6414. key.offset += num_bytes;
  6415. goto skip;
  6416. }
  6417. if (ordered)
  6418. btrfs_put_ordered_extent(ordered);
  6419. extent_locked = 1;
  6420. continue;
  6421. }
  6422. if (nr_extents == 1) {
  6423. /* update extent pointer in place */
  6424. btrfs_set_file_extent_disk_bytenr(leaf, fi,
  6425. new_extents[0].disk_bytenr);
  6426. btrfs_set_file_extent_disk_num_bytes(leaf, fi,
  6427. new_extents[0].disk_num_bytes);
  6428. btrfs_mark_buffer_dirty(leaf);
  6429. btrfs_drop_extent_cache(inode, key.offset,
  6430. key.offset + num_bytes - 1, 0);
  6431. ret = btrfs_inc_extent_ref(trans, root,
  6432. new_extents[0].disk_bytenr,
  6433. new_extents[0].disk_num_bytes,
  6434. leaf->start,
  6435. root->root_key.objectid,
  6436. trans->transid,
  6437. key.objectid);
  6438. BUG_ON(ret);
  6439. ret = btrfs_free_extent(trans, root,
  6440. extent_key->objectid,
  6441. extent_key->offset,
  6442. leaf->start,
  6443. btrfs_header_owner(leaf),
  6444. btrfs_header_generation(leaf),
  6445. key.objectid, 0);
  6446. BUG_ON(ret);
  6447. btrfs_release_path(root, path);
  6448. key.offset += num_bytes;
  6449. } else {
  6450. BUG_ON(1);
  6451. #if 0
  6452. u64 alloc_hint;
  6453. u64 extent_len;
  6454. int i;
  6455. /*
  6456. * drop old extent pointer at first, then insert the
  6457. * new pointers one bye one
  6458. */
  6459. btrfs_release_path(root, path);
  6460. ret = btrfs_drop_extents(trans, root, inode, key.offset,
  6461. key.offset + num_bytes,
  6462. key.offset, &alloc_hint);
  6463. BUG_ON(ret);
  6464. for (i = 0; i < nr_extents; i++) {
  6465. if (ext_offset >= new_extents[i].num_bytes) {
  6466. ext_offset -= new_extents[i].num_bytes;
  6467. continue;
  6468. }
  6469. extent_len = min(new_extents[i].num_bytes -
  6470. ext_offset, num_bytes);
  6471. ret = btrfs_insert_empty_item(trans, root,
  6472. path, &key,
  6473. sizeof(*fi));
  6474. BUG_ON(ret);
  6475. leaf = path->nodes[0];
  6476. fi = btrfs_item_ptr(leaf, path->slots[0],
  6477. struct btrfs_file_extent_item);
  6478. btrfs_set_file_extent_generation(leaf, fi,
  6479. trans->transid);
  6480. btrfs_set_file_extent_type(leaf, fi,
  6481. BTRFS_FILE_EXTENT_REG);
  6482. btrfs_set_file_extent_disk_bytenr(leaf, fi,
  6483. new_extents[i].disk_bytenr);
  6484. btrfs_set_file_extent_disk_num_bytes(leaf, fi,
  6485. new_extents[i].disk_num_bytes);
  6486. btrfs_set_file_extent_ram_bytes(leaf, fi,
  6487. new_extents[i].ram_bytes);
  6488. btrfs_set_file_extent_compression(leaf, fi,
  6489. new_extents[i].compression);
  6490. btrfs_set_file_extent_encryption(leaf, fi,
  6491. new_extents[i].encryption);
  6492. btrfs_set_file_extent_other_encoding(leaf, fi,
  6493. new_extents[i].other_encoding);
  6494. btrfs_set_file_extent_num_bytes(leaf, fi,
  6495. extent_len);
  6496. ext_offset += new_extents[i].offset;
  6497. btrfs_set_file_extent_offset(leaf, fi,
  6498. ext_offset);
  6499. btrfs_mark_buffer_dirty(leaf);
  6500. btrfs_drop_extent_cache(inode, key.offset,
  6501. key.offset + extent_len - 1, 0);
  6502. ret = btrfs_inc_extent_ref(trans, root,
  6503. new_extents[i].disk_bytenr,
  6504. new_extents[i].disk_num_bytes,
  6505. leaf->start,
  6506. root->root_key.objectid,
  6507. trans->transid, key.objectid);
  6508. BUG_ON(ret);
  6509. btrfs_release_path(root, path);
  6510. inode_add_bytes(inode, extent_len);
  6511. ext_offset = 0;
  6512. num_bytes -= extent_len;
  6513. key.offset += extent_len;
  6514. if (num_bytes == 0)
  6515. break;
  6516. }
  6517. BUG_ON(i >= nr_extents);
  6518. #endif
  6519. }
  6520. if (extent_locked) {
  6521. unlock_extent(&BTRFS_I(inode)->io_tree, lock_start,
  6522. lock_end, GFP_NOFS);
  6523. extent_locked = 0;
  6524. }
  6525. skip:
  6526. if (ref_path->owner_objectid != BTRFS_MULTIPLE_OBJECTIDS &&
  6527. key.offset >= search_end)
  6528. break;
  6529. cond_resched();
  6530. }
  6531. ret = 0;
  6532. out:
  6533. btrfs_release_path(root, path);
  6534. if (inode) {
  6535. mutex_unlock(&inode->i_mutex);
  6536. if (extent_locked) {
  6537. unlock_extent(&BTRFS_I(inode)->io_tree, lock_start,
  6538. lock_end, GFP_NOFS);
  6539. }
  6540. iput(inode);
  6541. }
  6542. return ret;
  6543. }
  6544. int btrfs_reloc_tree_cache_ref(struct btrfs_trans_handle *trans,
  6545. struct btrfs_root *root,
  6546. struct extent_buffer *buf, u64 orig_start)
  6547. {
  6548. int level;
  6549. int ret;
  6550. BUG_ON(btrfs_header_generation(buf) != trans->transid);
  6551. BUG_ON(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID);
  6552. level = btrfs_header_level(buf);
  6553. if (level == 0) {
  6554. struct btrfs_leaf_ref *ref;
  6555. struct btrfs_leaf_ref *orig_ref;
  6556. orig_ref = btrfs_lookup_leaf_ref(root, orig_start);
  6557. if (!orig_ref)
  6558. return -ENOENT;
  6559. ref = btrfs_alloc_leaf_ref(root, orig_ref->nritems);
  6560. if (!ref) {
  6561. btrfs_free_leaf_ref(root, orig_ref);
  6562. return -ENOMEM;
  6563. }
  6564. ref->nritems = orig_ref->nritems;
  6565. memcpy(ref->extents, orig_ref->extents,
  6566. sizeof(ref->extents[0]) * ref->nritems);
  6567. btrfs_free_leaf_ref(root, orig_ref);
  6568. ref->root_gen = trans->transid;
  6569. ref->bytenr = buf->start;
  6570. ref->owner = btrfs_header_owner(buf);
  6571. ref->generation = btrfs_header_generation(buf);
  6572. ret = btrfs_add_leaf_ref(root, ref, 0);
  6573. WARN_ON(ret);
  6574. btrfs_free_leaf_ref(root, ref);
  6575. }
  6576. return 0;
  6577. }
  6578. static noinline int invalidate_extent_cache(struct btrfs_root *root,
  6579. struct extent_buffer *leaf,
  6580. struct btrfs_block_group_cache *group,
  6581. struct btrfs_root *target_root)
  6582. {
  6583. struct btrfs_key key;
  6584. struct inode *inode = NULL;
  6585. struct btrfs_file_extent_item *fi;
  6586. struct extent_state *cached_state = NULL;
  6587. u64 num_bytes;
  6588. u64 skip_objectid = 0;
  6589. u32 nritems;
  6590. u32 i;
  6591. nritems = btrfs_header_nritems(leaf);
  6592. for (i = 0; i < nritems; i++) {
  6593. btrfs_item_key_to_cpu(leaf, &key, i);
  6594. if (key.objectid == skip_objectid ||
  6595. key.type != BTRFS_EXTENT_DATA_KEY)
  6596. continue;
  6597. fi = btrfs_item_ptr(leaf, i, struct btrfs_file_extent_item);
  6598. if (btrfs_file_extent_type(leaf, fi) ==
  6599. BTRFS_FILE_EXTENT_INLINE)
  6600. continue;
  6601. if (btrfs_file_extent_disk_bytenr(leaf, fi) == 0)
  6602. continue;
  6603. if (!inode || btrfs_ino(inode) != key.objectid) {
  6604. iput(inode);
  6605. inode = btrfs_ilookup(target_root->fs_info->sb,
  6606. key.objectid, target_root, 1);
  6607. }
  6608. if (!inode) {
  6609. skip_objectid = key.objectid;
  6610. continue;
  6611. }
  6612. num_bytes = btrfs_file_extent_num_bytes(leaf, fi);
  6613. lock_extent_bits(&BTRFS_I(inode)->io_tree, key.offset,
  6614. key.offset + num_bytes - 1, 0, &cached_state,
  6615. GFP_NOFS);
  6616. btrfs_drop_extent_cache(inode, key.offset,
  6617. key.offset + num_bytes - 1, 1);
  6618. unlock_extent_cached(&BTRFS_I(inode)->io_tree, key.offset,
  6619. key.offset + num_bytes - 1, &cached_state,
  6620. GFP_NOFS);
  6621. cond_resched();
  6622. }
  6623. iput(inode);
  6624. return 0;
  6625. }
  6626. static noinline int replace_extents_in_leaf(struct btrfs_trans_handle *trans,
  6627. struct btrfs_root *root,
  6628. struct extent_buffer *leaf,
  6629. struct btrfs_block_group_cache *group,
  6630. struct inode *reloc_inode)
  6631. {
  6632. struct btrfs_key key;
  6633. struct btrfs_key extent_key;
  6634. struct btrfs_file_extent_item *fi;
  6635. struct btrfs_leaf_ref *ref;
  6636. struct disk_extent *new_extent;
  6637. u64 bytenr;
  6638. u64 num_bytes;
  6639. u32 nritems;
  6640. u32 i;
  6641. int ext_index;
  6642. int nr_extent;
  6643. int ret;
  6644. new_extent = kmalloc(sizeof(*new_extent), GFP_NOFS);
  6645. if (!new_extent)
  6646. return -ENOMEM;
  6647. ref = btrfs_lookup_leaf_ref(root, leaf->start);
  6648. BUG_ON(!ref);
  6649. ext_index = -1;
  6650. nritems = btrfs_header_nritems(leaf);
  6651. for (i = 0; i < nritems; i++) {
  6652. btrfs_item_key_to_cpu(leaf, &key, i);
  6653. if (btrfs_key_type(&key) != BTRFS_EXTENT_DATA_KEY)
  6654. continue;
  6655. fi = btrfs_item_ptr(leaf, i, struct btrfs_file_extent_item);
  6656. if (btrfs_file_extent_type(leaf, fi) ==
  6657. BTRFS_FILE_EXTENT_INLINE)
  6658. continue;
  6659. bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
  6660. num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
  6661. if (bytenr == 0)
  6662. continue;
  6663. ext_index++;
  6664. if (bytenr >= group->key.objectid + group->key.offset ||
  6665. bytenr + num_bytes <= group->key.objectid)
  6666. continue;
  6667. extent_key.objectid = bytenr;
  6668. extent_key.offset = num_bytes;
  6669. extent_key.type = BTRFS_EXTENT_ITEM_KEY;
  6670. nr_extent = 1;
  6671. ret = get_new_locations(reloc_inode, &extent_key,
  6672. group->key.objectid, 1,
  6673. &new_extent, &nr_extent);
  6674. if (ret > 0)
  6675. continue;
  6676. BUG_ON(ret < 0);
  6677. BUG_ON(ref->extents[ext_index].bytenr != bytenr);
  6678. BUG_ON(ref->extents[ext_index].num_bytes != num_bytes);
  6679. ref->extents[ext_index].bytenr = new_extent->disk_bytenr;
  6680. ref->extents[ext_index].num_bytes = new_extent->disk_num_bytes;
  6681. btrfs_set_file_extent_disk_bytenr(leaf, fi,
  6682. new_extent->disk_bytenr);
  6683. btrfs_set_file_extent_disk_num_bytes(leaf, fi,
  6684. new_extent->disk_num_bytes);
  6685. btrfs_mark_buffer_dirty(leaf);
  6686. ret = btrfs_inc_extent_ref(trans, root,
  6687. new_extent->disk_bytenr,
  6688. new_extent->disk_num_bytes,
  6689. leaf->start,
  6690. root->root_key.objectid,
  6691. trans->transid, key.objectid);
  6692. BUG_ON(ret);
  6693. ret = btrfs_free_extent(trans, root,
  6694. bytenr, num_bytes, leaf->start,
  6695. btrfs_header_owner(leaf),
  6696. btrfs_header_generation(leaf),
  6697. key.objectid, 0);
  6698. BUG_ON(ret);
  6699. cond_resched();
  6700. }
  6701. kfree(new_extent);
  6702. BUG_ON(ext_index + 1 != ref->nritems);
  6703. btrfs_free_leaf_ref(root, ref);
  6704. return 0;
  6705. }
  6706. int btrfs_free_reloc_root(struct btrfs_trans_handle *trans,
  6707. struct btrfs_root *root)
  6708. {
  6709. struct btrfs_root *reloc_root;
  6710. int ret;
  6711. if (root->reloc_root) {
  6712. reloc_root = root->reloc_root;
  6713. root->reloc_root = NULL;
  6714. list_add(&reloc_root->dead_list,
  6715. &root->fs_info->dead_reloc_roots);
  6716. btrfs_set_root_bytenr(&reloc_root->root_item,
  6717. reloc_root->node->start);
  6718. btrfs_set_root_level(&root->root_item,
  6719. btrfs_header_level(reloc_root->node));
  6720. memset(&reloc_root->root_item.drop_progress, 0,
  6721. sizeof(struct btrfs_disk_key));
  6722. reloc_root->root_item.drop_level = 0;
  6723. ret = btrfs_update_root(trans, root->fs_info->tree_root,
  6724. &reloc_root->root_key,
  6725. &reloc_root->root_item);
  6726. BUG_ON(ret);
  6727. }
  6728. return 0;
  6729. }
  6730. int btrfs_drop_dead_reloc_roots(struct btrfs_root *root)
  6731. {
  6732. struct btrfs_trans_handle *trans;
  6733. struct btrfs_root *reloc_root;
  6734. struct btrfs_root *prev_root = NULL;
  6735. struct list_head dead_roots;
  6736. int ret;
  6737. unsigned long nr;
  6738. INIT_LIST_HEAD(&dead_roots);
  6739. list_splice_init(&root->fs_info->dead_reloc_roots, &dead_roots);
  6740. while (!list_empty(&dead_roots)) {
  6741. reloc_root = list_entry(dead_roots.prev,
  6742. struct btrfs_root, dead_list);
  6743. list_del_init(&reloc_root->dead_list);
  6744. BUG_ON(reloc_root->commit_root != NULL);
  6745. while (1) {
  6746. trans = btrfs_join_transaction(root, 1);
  6747. BUG_ON(IS_ERR(trans));
  6748. mutex_lock(&root->fs_info->drop_mutex);
  6749. ret = btrfs_drop_snapshot(trans, reloc_root);
  6750. if (ret != -EAGAIN)
  6751. break;
  6752. mutex_unlock(&root->fs_info->drop_mutex);
  6753. nr = trans->blocks_used;
  6754. ret = btrfs_end_transaction(trans, root);
  6755. BUG_ON(ret);
  6756. btrfs_btree_balance_dirty(root, nr);
  6757. }
  6758. free_extent_buffer(reloc_root->node);
  6759. ret = btrfs_del_root(trans, root->fs_info->tree_root,
  6760. &reloc_root->root_key);
  6761. BUG_ON(ret);
  6762. mutex_unlock(&root->fs_info->drop_mutex);
  6763. nr = trans->blocks_used;
  6764. ret = btrfs_end_transaction(trans, root);
  6765. BUG_ON(ret);
  6766. btrfs_btree_balance_dirty(root, nr);
  6767. kfree(prev_root);
  6768. prev_root = reloc_root;
  6769. }
  6770. if (prev_root) {
  6771. btrfs_remove_leaf_refs(prev_root, (u64)-1, 0);
  6772. kfree(prev_root);
  6773. }
  6774. return 0;
  6775. }
  6776. int btrfs_add_dead_reloc_root(struct btrfs_root *root)
  6777. {
  6778. list_add(&root->dead_list, &root->fs_info->dead_reloc_roots);
  6779. return 0;
  6780. }
  6781. int btrfs_cleanup_reloc_trees(struct btrfs_root *root)
  6782. {
  6783. struct btrfs_root *reloc_root;
  6784. struct btrfs_trans_handle *trans;
  6785. struct btrfs_key location;
  6786. int found;
  6787. int ret;
  6788. mutex_lock(&root->fs_info->tree_reloc_mutex);
  6789. ret = btrfs_find_dead_roots(root, BTRFS_TREE_RELOC_OBJECTID, NULL);
  6790. BUG_ON(ret);
  6791. found = !list_empty(&root->fs_info->dead_reloc_roots);
  6792. mutex_unlock(&root->fs_info->tree_reloc_mutex);
  6793. if (found) {
  6794. trans = btrfs_start_transaction(root, 1);
  6795. BUG_ON(IS_ERR(trans));
  6796. ret = btrfs_commit_transaction(trans, root);
  6797. BUG_ON(ret);
  6798. }
  6799. location.objectid = BTRFS_DATA_RELOC_TREE_OBJECTID;
  6800. location.offset = (u64)-1;
  6801. location.type = BTRFS_ROOT_ITEM_KEY;
  6802. reloc_root = btrfs_read_fs_root_no_name(root->fs_info, &location);
  6803. BUG_ON(!reloc_root);
  6804. ret = btrfs_orphan_cleanup(reloc_root);
  6805. BUG_ON(ret);
  6806. return 0;
  6807. }
  6808. static noinline int init_reloc_tree(struct btrfs_trans_handle *trans,
  6809. struct btrfs_root *root)
  6810. {
  6811. struct btrfs_root *reloc_root;
  6812. struct extent_buffer *eb;
  6813. struct btrfs_root_item *root_item;
  6814. struct btrfs_key root_key;
  6815. int ret;
  6816. BUG_ON(!root->ref_cows);
  6817. if (root->reloc_root)
  6818. return 0;
  6819. root_item = kmalloc(sizeof(*root_item), GFP_NOFS);
  6820. if (!root_item)
  6821. return -ENOMEM;
  6822. ret = btrfs_copy_root(trans, root, root->commit_root,
  6823. &eb, BTRFS_TREE_RELOC_OBJECTID);
  6824. BUG_ON(ret);
  6825. root_key.objectid = BTRFS_TREE_RELOC_OBJECTID;
  6826. root_key.offset = root->root_key.objectid;
  6827. root_key.type = BTRFS_ROOT_ITEM_KEY;
  6828. memcpy(root_item, &root->root_item, sizeof(root_item));
  6829. btrfs_set_root_refs(root_item, 0);
  6830. btrfs_set_root_bytenr(root_item, eb->start);
  6831. btrfs_set_root_level(root_item, btrfs_header_level(eb));
  6832. btrfs_set_root_generation(root_item, trans->transid);
  6833. btrfs_tree_unlock(eb);
  6834. free_extent_buffer(eb);
  6835. ret = btrfs_insert_root(trans, root->fs_info->tree_root,
  6836. &root_key, root_item);
  6837. BUG_ON(ret);
  6838. kfree(root_item);
  6839. reloc_root = btrfs_read_fs_root_no_radix(root->fs_info->tree_root,
  6840. &root_key);
  6841. BUG_ON(IS_ERR(reloc_root));
  6842. reloc_root->last_trans = trans->transid;
  6843. reloc_root->commit_root = NULL;
  6844. reloc_root->ref_tree = &root->fs_info->reloc_ref_tree;
  6845. root->reloc_root = reloc_root;
  6846. return 0;
  6847. }
  6848. /*
  6849. * Core function of space balance.
  6850. *
  6851. * The idea is using reloc trees to relocate tree blocks in reference
  6852. * counted roots. There is one reloc tree for each subvol, and all
  6853. * reloc trees share same root key objectid. Reloc trees are snapshots
  6854. * of the latest committed roots of subvols (root->commit_root).
  6855. *
  6856. * To relocate a tree block referenced by a subvol, there are two steps.
  6857. * COW the block through subvol's reloc tree, then update block pointer
  6858. * in the subvol to point to the new block. Since all reloc trees share
  6859. * same root key objectid, doing special handing for tree blocks owned
  6860. * by them is easy. Once a tree block has been COWed in one reloc tree,
  6861. * we can use the resulting new block directly when the same block is
  6862. * required to COW again through other reloc trees. By this way, relocated
  6863. * tree blocks are shared between reloc trees, so they are also shared
  6864. * between subvols.
  6865. */
  6866. static noinline int relocate_one_path(struct btrfs_trans_handle *trans,
  6867. struct btrfs_root *root,
  6868. struct btrfs_path *path,
  6869. struct btrfs_key *first_key,
  6870. struct btrfs_ref_path *ref_path,
  6871. struct btrfs_block_group_cache *group,
  6872. struct inode *reloc_inode)
  6873. {
  6874. struct btrfs_root *reloc_root;
  6875. struct extent_buffer *eb = NULL;
  6876. struct btrfs_key *keys;
  6877. u64 *nodes;
  6878. int level;
  6879. int shared_level;
  6880. int lowest_level = 0;
  6881. int ret;
  6882. if (ref_path->owner_objectid < BTRFS_FIRST_FREE_OBJECTID)
  6883. lowest_level = ref_path->owner_objectid;
  6884. if (!root->ref_cows) {
  6885. path->lowest_level = lowest_level;
  6886. ret = btrfs_search_slot(trans, root, first_key, path, 0, 1);
  6887. BUG_ON(ret < 0);
  6888. path->lowest_level = 0;
  6889. btrfs_release_path(root, path);
  6890. return 0;
  6891. }
  6892. mutex_lock(&root->fs_info->tree_reloc_mutex);
  6893. ret = init_reloc_tree(trans, root);
  6894. BUG_ON(ret);
  6895. reloc_root = root->reloc_root;
  6896. shared_level = ref_path->shared_level;
  6897. ref_path->shared_level = BTRFS_MAX_LEVEL - 1;
  6898. keys = ref_path->node_keys;
  6899. nodes = ref_path->new_nodes;
  6900. memset(&keys[shared_level + 1], 0,
  6901. sizeof(*keys) * (BTRFS_MAX_LEVEL - shared_level - 1));
  6902. memset(&nodes[shared_level + 1], 0,
  6903. sizeof(*nodes) * (BTRFS_MAX_LEVEL - shared_level - 1));
  6904. if (nodes[lowest_level] == 0) {
  6905. path->lowest_level = lowest_level;
  6906. ret = btrfs_search_slot(trans, reloc_root, first_key, path,
  6907. 0, 1);
  6908. BUG_ON(ret);
  6909. for (level = lowest_level; level < BTRFS_MAX_LEVEL; level++) {
  6910. eb = path->nodes[level];
  6911. if (!eb || eb == reloc_root->node)
  6912. break;
  6913. nodes[level] = eb->start;
  6914. if (level == 0)
  6915. btrfs_item_key_to_cpu(eb, &keys[level], 0);
  6916. else
  6917. btrfs_node_key_to_cpu(eb, &keys[level], 0);
  6918. }
  6919. if (nodes[0] &&
  6920. ref_path->owner_objectid >= BTRFS_FIRST_FREE_OBJECTID) {
  6921. eb = path->nodes[0];
  6922. ret = replace_extents_in_leaf(trans, reloc_root, eb,
  6923. group, reloc_inode);
  6924. BUG_ON(ret);
  6925. }
  6926. btrfs_release_path(reloc_root, path);
  6927. } else {
  6928. ret = btrfs_merge_path(trans, reloc_root, keys, nodes,
  6929. lowest_level);
  6930. BUG_ON(ret);
  6931. }
  6932. /*
  6933. * replace tree blocks in the fs tree with tree blocks in
  6934. * the reloc tree.
  6935. */
  6936. ret = btrfs_merge_path(trans, root, keys, nodes, lowest_level);
  6937. BUG_ON(ret < 0);
  6938. if (ref_path->owner_objectid >= BTRFS_FIRST_FREE_OBJECTID) {
  6939. ret = btrfs_search_slot(trans, reloc_root, first_key, path,
  6940. 0, 0);
  6941. BUG_ON(ret);
  6942. extent_buffer_get(path->nodes[0]);
  6943. eb = path->nodes[0];
  6944. btrfs_release_path(reloc_root, path);
  6945. ret = invalidate_extent_cache(reloc_root, eb, group, root);
  6946. BUG_ON(ret);
  6947. free_extent_buffer(eb);
  6948. }
  6949. mutex_unlock(&root->fs_info->tree_reloc_mutex);
  6950. path->lowest_level = 0;
  6951. return 0;
  6952. }
  6953. static noinline int relocate_tree_block(struct btrfs_trans_handle *trans,
  6954. struct btrfs_root *root,
  6955. struct btrfs_path *path,
  6956. struct btrfs_key *first_key,
  6957. struct btrfs_ref_path *ref_path)
  6958. {
  6959. int ret;
  6960. ret = relocate_one_path(trans, root, path, first_key,
  6961. ref_path, NULL, NULL);
  6962. BUG_ON(ret);
  6963. return 0;
  6964. }
  6965. static noinline int del_extent_zero(struct btrfs_trans_handle *trans,
  6966. struct btrfs_root *extent_root,
  6967. struct btrfs_path *path,
  6968. struct btrfs_key *extent_key)
  6969. {
  6970. int ret;
  6971. ret = btrfs_search_slot(trans, extent_root, extent_key, path, -1, 1);
  6972. if (ret)
  6973. goto out;
  6974. ret = btrfs_del_item(trans, extent_root, path);
  6975. out:
  6976. btrfs_release_path(extent_root, path);
  6977. return ret;
  6978. }
  6979. static noinline struct btrfs_root *read_ref_root(struct btrfs_fs_info *fs_info,
  6980. struct btrfs_ref_path *ref_path)
  6981. {
  6982. struct btrfs_key root_key;
  6983. root_key.objectid = ref_path->root_objectid;
  6984. root_key.type = BTRFS_ROOT_ITEM_KEY;
  6985. if (is_cowonly_root(ref_path->root_objectid))
  6986. root_key.offset = 0;
  6987. else
  6988. root_key.offset = (u64)-1;
  6989. return btrfs_read_fs_root_no_name(fs_info, &root_key);
  6990. }
  6991. static noinline int relocate_one_extent(struct btrfs_root *extent_root,
  6992. struct btrfs_path *path,
  6993. struct btrfs_key *extent_key,
  6994. struct btrfs_block_group_cache *group,
  6995. struct inode *reloc_inode, int pass)
  6996. {
  6997. struct btrfs_trans_handle *trans;
  6998. struct btrfs_root *found_root;
  6999. struct btrfs_ref_path *ref_path = NULL;
  7000. struct disk_extent *new_extents = NULL;
  7001. int nr_extents = 0;
  7002. int loops;
  7003. int ret;
  7004. int level;
  7005. struct btrfs_key first_key;
  7006. u64 prev_block = 0;
  7007. trans = btrfs_start_transaction(extent_root, 1);
  7008. BUG_ON(IS_ERR(trans));
  7009. if (extent_key->objectid == 0) {
  7010. ret = del_extent_zero(trans, extent_root, path, extent_key);
  7011. goto out;
  7012. }
  7013. ref_path = kmalloc(sizeof(*ref_path), GFP_NOFS);
  7014. if (!ref_path) {
  7015. ret = -ENOMEM;
  7016. goto out;
  7017. }
  7018. for (loops = 0; ; loops++) {
  7019. if (loops == 0) {
  7020. ret = btrfs_first_ref_path(trans, extent_root, ref_path,
  7021. extent_key->objectid);
  7022. } else {
  7023. ret = btrfs_next_ref_path(trans, extent_root, ref_path);
  7024. }
  7025. if (ret < 0)
  7026. goto out;
  7027. if (ret > 0)
  7028. break;
  7029. if (ref_path->root_objectid == BTRFS_TREE_LOG_OBJECTID ||
  7030. ref_path->root_objectid == BTRFS_TREE_RELOC_OBJECTID)
  7031. continue;
  7032. found_root = read_ref_root(extent_root->fs_info, ref_path);
  7033. BUG_ON(!found_root);
  7034. /*
  7035. * for reference counted tree, only process reference paths
  7036. * rooted at the latest committed root.
  7037. */
  7038. if (found_root->ref_cows &&
  7039. ref_path->root_generation != found_root->root_key.offset)
  7040. continue;
  7041. if (ref_path->owner_objectid >= BTRFS_FIRST_FREE_OBJECTID) {
  7042. if (pass == 0) {
  7043. /*
  7044. * copy data extents to new locations
  7045. */
  7046. u64 group_start = group->key.objectid;
  7047. ret = relocate_data_extent(reloc_inode,
  7048. extent_key,
  7049. group_start);
  7050. if (ret < 0)
  7051. goto out;
  7052. break;
  7053. }
  7054. level = 0;
  7055. } else {
  7056. level = ref_path->owner_objectid;
  7057. }
  7058. if (prev_block != ref_path->nodes[level]) {
  7059. struct extent_buffer *eb;
  7060. u64 block_start = ref_path->nodes[level];
  7061. u64 block_size = btrfs_level_size(found_root, level);
  7062. eb = read_tree_block(found_root, block_start,
  7063. block_size, 0);
  7064. if (!eb) {
  7065. ret = -EIO;
  7066. goto out;
  7067. }
  7068. btrfs_tree_lock(eb);
  7069. BUG_ON(level != btrfs_header_level(eb));
  7070. if (level == 0)
  7071. btrfs_item_key_to_cpu(eb, &first_key, 0);
  7072. else
  7073. btrfs_node_key_to_cpu(eb, &first_key, 0);
  7074. btrfs_tree_unlock(eb);
  7075. free_extent_buffer(eb);
  7076. prev_block = block_start;
  7077. }
  7078. mutex_lock(&extent_root->fs_info->trans_mutex);
  7079. btrfs_record_root_in_trans(found_root);
  7080. mutex_unlock(&extent_root->fs_info->trans_mutex);
  7081. if (ref_path->owner_objectid >= BTRFS_FIRST_FREE_OBJECTID) {
  7082. /*
  7083. * try to update data extent references while
  7084. * keeping metadata shared between snapshots.
  7085. */
  7086. if (pass == 1) {
  7087. ret = relocate_one_path(trans, found_root,
  7088. path, &first_key, ref_path,
  7089. group, reloc_inode);
  7090. if (ret < 0)
  7091. goto out;
  7092. continue;
  7093. }
  7094. /*
  7095. * use fallback method to process the remaining
  7096. * references.
  7097. */
  7098. if (!new_extents) {
  7099. u64 group_start = group->key.objectid;
  7100. new_extents = kmalloc(sizeof(*new_extents),
  7101. GFP_NOFS);
  7102. nr_extents = 1;
  7103. ret = get_new_locations(reloc_inode,
  7104. extent_key,
  7105. group_start, 1,
  7106. &new_extents,
  7107. &nr_extents);
  7108. if (ret)
  7109. goto out;
  7110. }
  7111. ret = replace_one_extent(trans, found_root,
  7112. path, extent_key,
  7113. &first_key, ref_path,
  7114. new_extents, nr_extents);
  7115. } else {
  7116. ret = relocate_tree_block(trans, found_root, path,
  7117. &first_key, ref_path);
  7118. }
  7119. if (ret < 0)
  7120. goto out;
  7121. }
  7122. ret = 0;
  7123. out:
  7124. btrfs_end_transaction(trans, extent_root);
  7125. kfree(new_extents);
  7126. kfree(ref_path);
  7127. return ret;
  7128. }
  7129. #endif
  7130. static u64 update_block_group_flags(struct btrfs_root *root, u64 flags)
  7131. {
  7132. u64 num_devices;
  7133. u64 stripped = BTRFS_BLOCK_GROUP_RAID0 |
  7134. BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10;
  7135. /*
  7136. * we add in the count of missing devices because we want
  7137. * to make sure that any RAID levels on a degraded FS
  7138. * continue to be honored.
  7139. */
  7140. num_devices = root->fs_info->fs_devices->rw_devices +
  7141. root->fs_info->fs_devices->missing_devices;
  7142. if (num_devices == 1) {
  7143. stripped |= BTRFS_BLOCK_GROUP_DUP;
  7144. stripped = flags & ~stripped;
  7145. /* turn raid0 into single device chunks */
  7146. if (flags & BTRFS_BLOCK_GROUP_RAID0)
  7147. return stripped;
  7148. /* turn mirroring into duplication */
  7149. if (flags & (BTRFS_BLOCK_GROUP_RAID1 |
  7150. BTRFS_BLOCK_GROUP_RAID10))
  7151. return stripped | BTRFS_BLOCK_GROUP_DUP;
  7152. return flags;
  7153. } else {
  7154. /* they already had raid on here, just return */
  7155. if (flags & stripped)
  7156. return flags;
  7157. stripped |= BTRFS_BLOCK_GROUP_DUP;
  7158. stripped = flags & ~stripped;
  7159. /* switch duplicated blocks with raid1 */
  7160. if (flags & BTRFS_BLOCK_GROUP_DUP)
  7161. return stripped | BTRFS_BLOCK_GROUP_RAID1;
  7162. /* turn single device chunks into raid0 */
  7163. return stripped | BTRFS_BLOCK_GROUP_RAID0;
  7164. }
  7165. return flags;
  7166. }
  7167. static int set_block_group_ro(struct btrfs_block_group_cache *cache)
  7168. {
  7169. struct btrfs_space_info *sinfo = cache->space_info;
  7170. u64 num_bytes;
  7171. int ret = -ENOSPC;
  7172. if (cache->ro)
  7173. return 0;
  7174. spin_lock(&sinfo->lock);
  7175. spin_lock(&cache->lock);
  7176. num_bytes = cache->key.offset - cache->reserved - cache->pinned -
  7177. cache->bytes_super - btrfs_block_group_used(&cache->item);
  7178. if (sinfo->bytes_used + sinfo->bytes_reserved + sinfo->bytes_pinned +
  7179. sinfo->bytes_may_use + sinfo->bytes_readonly +
  7180. cache->reserved_pinned + num_bytes <= sinfo->total_bytes) {
  7181. sinfo->bytes_readonly += num_bytes;
  7182. sinfo->bytes_reserved += cache->reserved_pinned;
  7183. cache->reserved_pinned = 0;
  7184. cache->ro = 1;
  7185. ret = 0;
  7186. }
  7187. spin_unlock(&cache->lock);
  7188. spin_unlock(&sinfo->lock);
  7189. return ret;
  7190. }
  7191. int btrfs_set_block_group_ro(struct btrfs_root *root,
  7192. struct btrfs_block_group_cache *cache)
  7193. {
  7194. struct btrfs_trans_handle *trans;
  7195. u64 alloc_flags;
  7196. int ret;
  7197. BUG_ON(cache->ro);
  7198. trans = btrfs_join_transaction(root, 1);
  7199. BUG_ON(IS_ERR(trans));
  7200. alloc_flags = update_block_group_flags(root, cache->flags);
  7201. if (alloc_flags != cache->flags)
  7202. do_chunk_alloc(trans, root, 2 * 1024 * 1024, alloc_flags,
  7203. CHUNK_ALLOC_FORCE);
  7204. ret = set_block_group_ro(cache);
  7205. if (!ret)
  7206. goto out;
  7207. alloc_flags = get_alloc_profile(root, cache->space_info->flags);
  7208. ret = do_chunk_alloc(trans, root, 2 * 1024 * 1024, alloc_flags,
  7209. CHUNK_ALLOC_FORCE);
  7210. if (ret < 0)
  7211. goto out;
  7212. ret = set_block_group_ro(cache);
  7213. out:
  7214. btrfs_end_transaction(trans, root);
  7215. return ret;
  7216. }
  7217. int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans,
  7218. struct btrfs_root *root, u64 type)
  7219. {
  7220. u64 alloc_flags = get_alloc_profile(root, type);
  7221. return do_chunk_alloc(trans, root, 2 * 1024 * 1024, alloc_flags,
  7222. CHUNK_ALLOC_FORCE);
  7223. }
  7224. /*
  7225. * helper to account the unused space of all the readonly block group in the
  7226. * list. takes mirrors into account.
  7227. */
  7228. static u64 __btrfs_get_ro_block_group_free_space(struct list_head *groups_list)
  7229. {
  7230. struct btrfs_block_group_cache *block_group;
  7231. u64 free_bytes = 0;
  7232. int factor;
  7233. list_for_each_entry(block_group, groups_list, list) {
  7234. spin_lock(&block_group->lock);
  7235. if (!block_group->ro) {
  7236. spin_unlock(&block_group->lock);
  7237. continue;
  7238. }
  7239. if (block_group->flags & (BTRFS_BLOCK_GROUP_RAID1 |
  7240. BTRFS_BLOCK_GROUP_RAID10 |
  7241. BTRFS_BLOCK_GROUP_DUP))
  7242. factor = 2;
  7243. else
  7244. factor = 1;
  7245. free_bytes += (block_group->key.offset -
  7246. btrfs_block_group_used(&block_group->item)) *
  7247. factor;
  7248. spin_unlock(&block_group->lock);
  7249. }
  7250. return free_bytes;
  7251. }
  7252. /*
  7253. * helper to account the unused space of all the readonly block group in the
  7254. * space_info. takes mirrors into account.
  7255. */
  7256. u64 btrfs_account_ro_block_groups_free_space(struct btrfs_space_info *sinfo)
  7257. {
  7258. int i;
  7259. u64 free_bytes = 0;
  7260. spin_lock(&sinfo->lock);
  7261. for(i = 0; i < BTRFS_NR_RAID_TYPES; i++)
  7262. if (!list_empty(&sinfo->block_groups[i]))
  7263. free_bytes += __btrfs_get_ro_block_group_free_space(
  7264. &sinfo->block_groups[i]);
  7265. spin_unlock(&sinfo->lock);
  7266. return free_bytes;
  7267. }
  7268. int btrfs_set_block_group_rw(struct btrfs_root *root,
  7269. struct btrfs_block_group_cache *cache)
  7270. {
  7271. struct btrfs_space_info *sinfo = cache->space_info;
  7272. u64 num_bytes;
  7273. BUG_ON(!cache->ro);
  7274. spin_lock(&sinfo->lock);
  7275. spin_lock(&cache->lock);
  7276. num_bytes = cache->key.offset - cache->reserved - cache->pinned -
  7277. cache->bytes_super - btrfs_block_group_used(&cache->item);
  7278. sinfo->bytes_readonly -= num_bytes;
  7279. cache->ro = 0;
  7280. spin_unlock(&cache->lock);
  7281. spin_unlock(&sinfo->lock);
  7282. return 0;
  7283. }
  7284. /*
  7285. * checks to see if its even possible to relocate this block group.
  7286. *
  7287. * @return - -1 if it's not a good idea to relocate this block group, 0 if its
  7288. * ok to go ahead and try.
  7289. */
  7290. int btrfs_can_relocate(struct btrfs_root *root, u64 bytenr)
  7291. {
  7292. struct btrfs_block_group_cache *block_group;
  7293. struct btrfs_space_info *space_info;
  7294. struct btrfs_fs_devices *fs_devices = root->fs_info->fs_devices;
  7295. struct btrfs_device *device;
  7296. int full = 0;
  7297. int ret = 0;
  7298. block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
  7299. /* odd, couldn't find the block group, leave it alone */
  7300. if (!block_group)
  7301. return -1;
  7302. /* no bytes used, we're good */
  7303. if (!btrfs_block_group_used(&block_group->item))
  7304. goto out;
  7305. space_info = block_group->space_info;
  7306. spin_lock(&space_info->lock);
  7307. full = space_info->full;
  7308. /*
  7309. * if this is the last block group we have in this space, we can't
  7310. * relocate it unless we're able to allocate a new chunk below.
  7311. *
  7312. * Otherwise, we need to make sure we have room in the space to handle
  7313. * all of the extents from this block group. If we can, we're good
  7314. */
  7315. if ((space_info->total_bytes != block_group->key.offset) &&
  7316. (space_info->bytes_used + space_info->bytes_reserved +
  7317. space_info->bytes_pinned + space_info->bytes_readonly +
  7318. btrfs_block_group_used(&block_group->item) <
  7319. space_info->total_bytes)) {
  7320. spin_unlock(&space_info->lock);
  7321. goto out;
  7322. }
  7323. spin_unlock(&space_info->lock);
  7324. /*
  7325. * ok we don't have enough space, but maybe we have free space on our
  7326. * devices to allocate new chunks for relocation, so loop through our
  7327. * alloc devices and guess if we have enough space. However, if we
  7328. * were marked as full, then we know there aren't enough chunks, and we
  7329. * can just return.
  7330. */
  7331. ret = -1;
  7332. if (full)
  7333. goto out;
  7334. mutex_lock(&root->fs_info->chunk_mutex);
  7335. list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
  7336. u64 min_free = btrfs_block_group_used(&block_group->item);
  7337. u64 dev_offset;
  7338. /*
  7339. * check to make sure we can actually find a chunk with enough
  7340. * space to fit our block group in.
  7341. */
  7342. if (device->total_bytes > device->bytes_used + min_free) {
  7343. ret = find_free_dev_extent(NULL, device, min_free,
  7344. &dev_offset, NULL);
  7345. if (!ret)
  7346. break;
  7347. ret = -1;
  7348. }
  7349. }
  7350. mutex_unlock(&root->fs_info->chunk_mutex);
  7351. out:
  7352. btrfs_put_block_group(block_group);
  7353. return ret;
  7354. }
  7355. static int find_first_block_group(struct btrfs_root *root,
  7356. struct btrfs_path *path, struct btrfs_key *key)
  7357. {
  7358. int ret = 0;
  7359. struct btrfs_key found_key;
  7360. struct extent_buffer *leaf;
  7361. int slot;
  7362. ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
  7363. if (ret < 0)
  7364. goto out;
  7365. while (1) {
  7366. slot = path->slots[0];
  7367. leaf = path->nodes[0];
  7368. if (slot >= btrfs_header_nritems(leaf)) {
  7369. ret = btrfs_next_leaf(root, path);
  7370. if (ret == 0)
  7371. continue;
  7372. if (ret < 0)
  7373. goto out;
  7374. break;
  7375. }
  7376. btrfs_item_key_to_cpu(leaf, &found_key, slot);
  7377. if (found_key.objectid >= key->objectid &&
  7378. found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) {
  7379. ret = 0;
  7380. goto out;
  7381. }
  7382. path->slots[0]++;
  7383. }
  7384. out:
  7385. return ret;
  7386. }
  7387. void btrfs_put_block_group_cache(struct btrfs_fs_info *info)
  7388. {
  7389. struct btrfs_block_group_cache *block_group;
  7390. u64 last = 0;
  7391. while (1) {
  7392. struct inode *inode;
  7393. block_group = btrfs_lookup_first_block_group(info, last);
  7394. while (block_group) {
  7395. spin_lock(&block_group->lock);
  7396. if (block_group->iref)
  7397. break;
  7398. spin_unlock(&block_group->lock);
  7399. block_group = next_block_group(info->tree_root,
  7400. block_group);
  7401. }
  7402. if (!block_group) {
  7403. if (last == 0)
  7404. break;
  7405. last = 0;
  7406. continue;
  7407. }
  7408. inode = block_group->inode;
  7409. block_group->iref = 0;
  7410. block_group->inode = NULL;
  7411. spin_unlock(&block_group->lock);
  7412. iput(inode);
  7413. last = block_group->key.objectid + block_group->key.offset;
  7414. btrfs_put_block_group(block_group);
  7415. }
  7416. }
  7417. int btrfs_free_block_groups(struct btrfs_fs_info *info)
  7418. {
  7419. struct btrfs_block_group_cache *block_group;
  7420. struct btrfs_space_info *space_info;
  7421. struct btrfs_caching_control *caching_ctl;
  7422. struct rb_node *n;
  7423. down_write(&info->extent_commit_sem);
  7424. while (!list_empty(&info->caching_block_groups)) {
  7425. caching_ctl = list_entry(info->caching_block_groups.next,
  7426. struct btrfs_caching_control, list);
  7427. list_del(&caching_ctl->list);
  7428. put_caching_control(caching_ctl);
  7429. }
  7430. up_write(&info->extent_commit_sem);
  7431. spin_lock(&info->block_group_cache_lock);
  7432. while ((n = rb_last(&info->block_group_cache_tree)) != NULL) {
  7433. block_group = rb_entry(n, struct btrfs_block_group_cache,
  7434. cache_node);
  7435. rb_erase(&block_group->cache_node,
  7436. &info->block_group_cache_tree);
  7437. spin_unlock(&info->block_group_cache_lock);
  7438. down_write(&block_group->space_info->groups_sem);
  7439. list_del(&block_group->list);
  7440. up_write(&block_group->space_info->groups_sem);
  7441. if (block_group->cached == BTRFS_CACHE_STARTED)
  7442. wait_block_group_cache_done(block_group);
  7443. /*
  7444. * We haven't cached this block group, which means we could
  7445. * possibly have excluded extents on this block group.
  7446. */
  7447. if (block_group->cached == BTRFS_CACHE_NO)
  7448. free_excluded_extents(info->extent_root, block_group);
  7449. btrfs_remove_free_space_cache(block_group);
  7450. btrfs_put_block_group(block_group);
  7451. spin_lock(&info->block_group_cache_lock);
  7452. }
  7453. spin_unlock(&info->block_group_cache_lock);
  7454. /* now that all the block groups are freed, go through and
  7455. * free all the space_info structs. This is only called during
  7456. * the final stages of unmount, and so we know nobody is
  7457. * using them. We call synchronize_rcu() once before we start,
  7458. * just to be on the safe side.
  7459. */
  7460. synchronize_rcu();
  7461. release_global_block_rsv(info);
  7462. while(!list_empty(&info->space_info)) {
  7463. space_info = list_entry(info->space_info.next,
  7464. struct btrfs_space_info,
  7465. list);
  7466. if (space_info->bytes_pinned > 0 ||
  7467. space_info->bytes_reserved > 0) {
  7468. WARN_ON(1);
  7469. dump_space_info(space_info, 0, 0);
  7470. }
  7471. list_del(&space_info->list);
  7472. kfree(space_info);
  7473. }
  7474. return 0;
  7475. }
  7476. static void __link_block_group(struct btrfs_space_info *space_info,
  7477. struct btrfs_block_group_cache *cache)
  7478. {
  7479. int index = get_block_group_index(cache);
  7480. down_write(&space_info->groups_sem);
  7481. list_add_tail(&cache->list, &space_info->block_groups[index]);
  7482. up_write(&space_info->groups_sem);
  7483. }
  7484. int btrfs_read_block_groups(struct btrfs_root *root)
  7485. {
  7486. struct btrfs_path *path;
  7487. int ret;
  7488. struct btrfs_block_group_cache *cache;
  7489. struct btrfs_fs_info *info = root->fs_info;
  7490. struct btrfs_space_info *space_info;
  7491. struct btrfs_key key;
  7492. struct btrfs_key found_key;
  7493. struct extent_buffer *leaf;
  7494. int need_clear = 0;
  7495. u64 cache_gen;
  7496. root = info->extent_root;
  7497. key.objectid = 0;
  7498. key.offset = 0;
  7499. btrfs_set_key_type(&key, BTRFS_BLOCK_GROUP_ITEM_KEY);
  7500. path = btrfs_alloc_path();
  7501. if (!path)
  7502. return -ENOMEM;
  7503. cache_gen = btrfs_super_cache_generation(&root->fs_info->super_copy);
  7504. if (cache_gen != 0 &&
  7505. btrfs_super_generation(&root->fs_info->super_copy) != cache_gen)
  7506. need_clear = 1;
  7507. if (btrfs_test_opt(root, CLEAR_CACHE))
  7508. need_clear = 1;
  7509. if (!btrfs_test_opt(root, SPACE_CACHE) && cache_gen)
  7510. printk(KERN_INFO "btrfs: disk space caching is enabled\n");
  7511. while (1) {
  7512. ret = find_first_block_group(root, path, &key);
  7513. if (ret > 0)
  7514. break;
  7515. if (ret != 0)
  7516. goto error;
  7517. leaf = path->nodes[0];
  7518. btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
  7519. cache = kzalloc(sizeof(*cache), GFP_NOFS);
  7520. if (!cache) {
  7521. ret = -ENOMEM;
  7522. goto error;
  7523. }
  7524. cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
  7525. GFP_NOFS);
  7526. if (!cache->free_space_ctl) {
  7527. kfree(cache);
  7528. ret = -ENOMEM;
  7529. goto error;
  7530. }
  7531. atomic_set(&cache->count, 1);
  7532. spin_lock_init(&cache->lock);
  7533. cache->fs_info = info;
  7534. INIT_LIST_HEAD(&cache->list);
  7535. INIT_LIST_HEAD(&cache->cluster_list);
  7536. if (need_clear)
  7537. cache->disk_cache_state = BTRFS_DC_CLEAR;
  7538. read_extent_buffer(leaf, &cache->item,
  7539. btrfs_item_ptr_offset(leaf, path->slots[0]),
  7540. sizeof(cache->item));
  7541. memcpy(&cache->key, &found_key, sizeof(found_key));
  7542. key.objectid = found_key.objectid + found_key.offset;
  7543. btrfs_release_path(root, path);
  7544. cache->flags = btrfs_block_group_flags(&cache->item);
  7545. cache->sectorsize = root->sectorsize;
  7546. btrfs_init_free_space_ctl(cache);
  7547. /*
  7548. * We need to exclude the super stripes now so that the space
  7549. * info has super bytes accounted for, otherwise we'll think
  7550. * we have more space than we actually do.
  7551. */
  7552. exclude_super_stripes(root, cache);
  7553. /*
  7554. * check for two cases, either we are full, and therefore
  7555. * don't need to bother with the caching work since we won't
  7556. * find any space, or we are empty, and we can just add all
  7557. * the space in and be done with it. This saves us _alot_ of
  7558. * time, particularly in the full case.
  7559. */
  7560. if (found_key.offset == btrfs_block_group_used(&cache->item)) {
  7561. cache->last_byte_to_unpin = (u64)-1;
  7562. cache->cached = BTRFS_CACHE_FINISHED;
  7563. free_excluded_extents(root, cache);
  7564. } else if (btrfs_block_group_used(&cache->item) == 0) {
  7565. cache->last_byte_to_unpin = (u64)-1;
  7566. cache->cached = BTRFS_CACHE_FINISHED;
  7567. add_new_free_space(cache, root->fs_info,
  7568. found_key.objectid,
  7569. found_key.objectid +
  7570. found_key.offset);
  7571. free_excluded_extents(root, cache);
  7572. }
  7573. ret = update_space_info(info, cache->flags, found_key.offset,
  7574. btrfs_block_group_used(&cache->item),
  7575. &space_info);
  7576. BUG_ON(ret);
  7577. cache->space_info = space_info;
  7578. spin_lock(&cache->space_info->lock);
  7579. cache->space_info->bytes_readonly += cache->bytes_super;
  7580. spin_unlock(&cache->space_info->lock);
  7581. __link_block_group(space_info, cache);
  7582. ret = btrfs_add_block_group_cache(root->fs_info, cache);
  7583. BUG_ON(ret);
  7584. set_avail_alloc_bits(root->fs_info, cache->flags);
  7585. if (btrfs_chunk_readonly(root, cache->key.objectid))
  7586. set_block_group_ro(cache);
  7587. }
  7588. list_for_each_entry_rcu(space_info, &root->fs_info->space_info, list) {
  7589. if (!(get_alloc_profile(root, space_info->flags) &
  7590. (BTRFS_BLOCK_GROUP_RAID10 |
  7591. BTRFS_BLOCK_GROUP_RAID1 |
  7592. BTRFS_BLOCK_GROUP_DUP)))
  7593. continue;
  7594. /*
  7595. * avoid allocating from un-mirrored block group if there are
  7596. * mirrored block groups.
  7597. */
  7598. list_for_each_entry(cache, &space_info->block_groups[3], list)
  7599. set_block_group_ro(cache);
  7600. list_for_each_entry(cache, &space_info->block_groups[4], list)
  7601. set_block_group_ro(cache);
  7602. }
  7603. init_global_block_rsv(info);
  7604. ret = 0;
  7605. error:
  7606. btrfs_free_path(path);
  7607. return ret;
  7608. }
  7609. int btrfs_make_block_group(struct btrfs_trans_handle *trans,
  7610. struct btrfs_root *root, u64 bytes_used,
  7611. u64 type, u64 chunk_objectid, u64 chunk_offset,
  7612. u64 size)
  7613. {
  7614. int ret;
  7615. struct btrfs_root *extent_root;
  7616. struct btrfs_block_group_cache *cache;
  7617. extent_root = root->fs_info->extent_root;
  7618. root->fs_info->last_trans_log_full_commit = trans->transid;
  7619. cache = kzalloc(sizeof(*cache), GFP_NOFS);
  7620. if (!cache)
  7621. return -ENOMEM;
  7622. cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
  7623. GFP_NOFS);
  7624. if (!cache->free_space_ctl) {
  7625. kfree(cache);
  7626. return -ENOMEM;
  7627. }
  7628. cache->key.objectid = chunk_offset;
  7629. cache->key.offset = size;
  7630. cache->key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
  7631. cache->sectorsize = root->sectorsize;
  7632. cache->fs_info = root->fs_info;
  7633. atomic_set(&cache->count, 1);
  7634. spin_lock_init(&cache->lock);
  7635. INIT_LIST_HEAD(&cache->list);
  7636. INIT_LIST_HEAD(&cache->cluster_list);
  7637. btrfs_init_free_space_ctl(cache);
  7638. btrfs_set_block_group_used(&cache->item, bytes_used);
  7639. btrfs_set_block_group_chunk_objectid(&cache->item, chunk_objectid);
  7640. cache->flags = type;
  7641. btrfs_set_block_group_flags(&cache->item, type);
  7642. cache->last_byte_to_unpin = (u64)-1;
  7643. cache->cached = BTRFS_CACHE_FINISHED;
  7644. exclude_super_stripes(root, cache);
  7645. add_new_free_space(cache, root->fs_info, chunk_offset,
  7646. chunk_offset + size);
  7647. free_excluded_extents(root, cache);
  7648. ret = update_space_info(root->fs_info, cache->flags, size, bytes_used,
  7649. &cache->space_info);
  7650. BUG_ON(ret);
  7651. spin_lock(&cache->space_info->lock);
  7652. cache->space_info->bytes_readonly += cache->bytes_super;
  7653. spin_unlock(&cache->space_info->lock);
  7654. __link_block_group(cache->space_info, cache);
  7655. ret = btrfs_add_block_group_cache(root->fs_info, cache);
  7656. BUG_ON(ret);
  7657. ret = btrfs_insert_item(trans, extent_root, &cache->key, &cache->item,
  7658. sizeof(cache->item));
  7659. BUG_ON(ret);
  7660. set_avail_alloc_bits(extent_root->fs_info, type);
  7661. return 0;
  7662. }
  7663. int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
  7664. struct btrfs_root *root, u64 group_start)
  7665. {
  7666. struct btrfs_path *path;
  7667. struct btrfs_block_group_cache *block_group;
  7668. struct btrfs_free_cluster *cluster;
  7669. struct btrfs_root *tree_root = root->fs_info->tree_root;
  7670. struct btrfs_key key;
  7671. struct inode *inode;
  7672. int ret;
  7673. int factor;
  7674. root = root->fs_info->extent_root;
  7675. block_group = btrfs_lookup_block_group(root->fs_info, group_start);
  7676. BUG_ON(!block_group);
  7677. BUG_ON(!block_group->ro);
  7678. /*
  7679. * Free the reserved super bytes from this block group before
  7680. * remove it.
  7681. */
  7682. free_excluded_extents(root, block_group);
  7683. memcpy(&key, &block_group->key, sizeof(key));
  7684. if (block_group->flags & (BTRFS_BLOCK_GROUP_DUP |
  7685. BTRFS_BLOCK_GROUP_RAID1 |
  7686. BTRFS_BLOCK_GROUP_RAID10))
  7687. factor = 2;
  7688. else
  7689. factor = 1;
  7690. /* make sure this block group isn't part of an allocation cluster */
  7691. cluster = &root->fs_info->data_alloc_cluster;
  7692. spin_lock(&cluster->refill_lock);
  7693. btrfs_return_cluster_to_free_space(block_group, cluster);
  7694. spin_unlock(&cluster->refill_lock);
  7695. /*
  7696. * make sure this block group isn't part of a metadata
  7697. * allocation cluster
  7698. */
  7699. cluster = &root->fs_info->meta_alloc_cluster;
  7700. spin_lock(&cluster->refill_lock);
  7701. btrfs_return_cluster_to_free_space(block_group, cluster);
  7702. spin_unlock(&cluster->refill_lock);
  7703. path = btrfs_alloc_path();
  7704. BUG_ON(!path);
  7705. inode = lookup_free_space_inode(root, block_group, path);
  7706. if (!IS_ERR(inode)) {
  7707. btrfs_orphan_add(trans, inode);
  7708. clear_nlink(inode);
  7709. /* One for the block groups ref */
  7710. spin_lock(&block_group->lock);
  7711. if (block_group->iref) {
  7712. block_group->iref = 0;
  7713. block_group->inode = NULL;
  7714. spin_unlock(&block_group->lock);
  7715. iput(inode);
  7716. } else {
  7717. spin_unlock(&block_group->lock);
  7718. }
  7719. /* One for our lookup ref */
  7720. iput(inode);
  7721. }
  7722. key.objectid = BTRFS_FREE_SPACE_OBJECTID;
  7723. key.offset = block_group->key.objectid;
  7724. key.type = 0;
  7725. ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1);
  7726. if (ret < 0)
  7727. goto out;
  7728. if (ret > 0)
  7729. btrfs_release_path(tree_root, path);
  7730. if (ret == 0) {
  7731. ret = btrfs_del_item(trans, tree_root, path);
  7732. if (ret)
  7733. goto out;
  7734. btrfs_release_path(tree_root, path);
  7735. }
  7736. spin_lock(&root->fs_info->block_group_cache_lock);
  7737. rb_erase(&block_group->cache_node,
  7738. &root->fs_info->block_group_cache_tree);
  7739. spin_unlock(&root->fs_info->block_group_cache_lock);
  7740. down_write(&block_group->space_info->groups_sem);
  7741. /*
  7742. * we must use list_del_init so people can check to see if they
  7743. * are still on the list after taking the semaphore
  7744. */
  7745. list_del_init(&block_group->list);
  7746. up_write(&block_group->space_info->groups_sem);
  7747. if (block_group->cached == BTRFS_CACHE_STARTED)
  7748. wait_block_group_cache_done(block_group);
  7749. btrfs_remove_free_space_cache(block_group);
  7750. spin_lock(&block_group->space_info->lock);
  7751. block_group->space_info->total_bytes -= block_group->key.offset;
  7752. block_group->space_info->bytes_readonly -= block_group->key.offset;
  7753. block_group->space_info->disk_total -= block_group->key.offset * factor;
  7754. spin_unlock(&block_group->space_info->lock);
  7755. memcpy(&key, &block_group->key, sizeof(key));
  7756. btrfs_clear_space_info_full(root->fs_info);
  7757. btrfs_put_block_group(block_group);
  7758. btrfs_put_block_group(block_group);
  7759. ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
  7760. if (ret > 0)
  7761. ret = -EIO;
  7762. if (ret < 0)
  7763. goto out;
  7764. ret = btrfs_del_item(trans, root, path);
  7765. out:
  7766. btrfs_free_path(path);
  7767. return ret;
  7768. }
  7769. int btrfs_init_space_info(struct btrfs_fs_info *fs_info)
  7770. {
  7771. struct btrfs_space_info *space_info;
  7772. int ret;
  7773. ret = update_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM, 0, 0,
  7774. &space_info);
  7775. if (ret)
  7776. return ret;
  7777. ret = update_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA, 0, 0,
  7778. &space_info);
  7779. if (ret)
  7780. return ret;
  7781. ret = update_space_info(fs_info, BTRFS_BLOCK_GROUP_DATA, 0, 0,
  7782. &space_info);
  7783. if (ret)
  7784. return ret;
  7785. return ret;
  7786. }
  7787. int btrfs_error_unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
  7788. {
  7789. return unpin_extent_range(root, start, end);
  7790. }
  7791. int btrfs_error_discard_extent(struct btrfs_root *root, u64 bytenr,
  7792. u64 num_bytes, u64 *actual_bytes)
  7793. {
  7794. return btrfs_discard_extent(root, bytenr, num_bytes, actual_bytes);
  7795. }
  7796. int btrfs_trim_fs(struct btrfs_root *root, struct fstrim_range *range)
  7797. {
  7798. struct btrfs_fs_info *fs_info = root->fs_info;
  7799. struct btrfs_block_group_cache *cache = NULL;
  7800. u64 group_trimmed;
  7801. u64 start;
  7802. u64 end;
  7803. u64 trimmed = 0;
  7804. int ret = 0;
  7805. cache = btrfs_lookup_block_group(fs_info, range->start);
  7806. while (cache) {
  7807. if (cache->key.objectid >= (range->start + range->len)) {
  7808. btrfs_put_block_group(cache);
  7809. break;
  7810. }
  7811. start = max(range->start, cache->key.objectid);
  7812. end = min(range->start + range->len,
  7813. cache->key.objectid + cache->key.offset);
  7814. if (end - start >= range->minlen) {
  7815. if (!block_group_cache_done(cache)) {
  7816. ret = cache_block_group(cache, NULL, root, 0);
  7817. if (!ret)
  7818. wait_block_group_cache_done(cache);
  7819. }
  7820. ret = btrfs_trim_block_group(cache,
  7821. &group_trimmed,
  7822. start,
  7823. end,
  7824. range->minlen);
  7825. trimmed += group_trimmed;
  7826. if (ret) {
  7827. btrfs_put_block_group(cache);
  7828. break;
  7829. }
  7830. }
  7831. cache = next_block_group(fs_info->tree_root, cache);
  7832. }
  7833. range->len = trimmed;
  7834. return ret;
  7835. }