extent-tree.c 200 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605660666076608660966106611661266136614661566166617661866196620662166226623662466256626662766286629663066316632663366346635663666376638663966406641664266436644664566466647664866496650665166526653665466556656665766586659666066616662666366646665666666676668666966706671667266736674667566766677667866796680668166826683668466856686668766886689669066916692669366946695669666976698669967006701670267036704670567066707670867096710671167126713671467156716671767186719672067216722672367246725672667276728672967306731673267336734673567366737673867396740674167426743674467456746674767486749675067516752675367546755675667576758675967606761676267636764676567666767676867696770677167726773677467756776677767786779678067816782678367846785678667876788678967906791679267936794679567966797679867996800680168026803680468056806680768086809681068116812681368146815681668176818681968206821682268236824682568266827682868296830683168326833683468356836683768386839684068416842684368446845684668476848684968506851685268536854685568566857685868596860686168626863686468656866686768686869687068716872687368746875687668776878687968806881688268836884688568866887688868896890689168926893689468956896689768986899690069016902690369046905690669076908690969106911691269136914691569166917691869196920692169226923692469256926692769286929693069316932693369346935693669376938693969406941694269436944694569466947694869496950695169526953695469556956695769586959696069616962696369646965696669676968696969706971697269736974697569766977697869796980698169826983698469856986698769886989699069916992699369946995699669976998699970007001700270037004700570067007700870097010701170127013701470157016701770187019702070217022702370247025702670277028702970307031703270337034703570367037703870397040704170427043704470457046704770487049705070517052705370547055705670577058705970607061706270637064706570667067706870697070707170727073707470757076707770787079708070817082708370847085708670877088708970907091709270937094709570967097709870997100710171027103710471057106710771087109711071117112711371147115711671177118711971207121712271237124712571267127712871297130713171327133713471357136713771387139714071417142714371447145714671477148714971507151715271537154715571567157715871597160716171627163716471657166716771687169717071717172717371747175717671777178717971807181718271837184718571867187718871897190719171927193719471957196719771987199720072017202720372047205720672077208720972107211721272137214721572167217721872197220722172227223722472257226722772287229723072317232723372347235723672377238723972407241724272437244724572467247724872497250725172527253725472557256725772587259726072617262726372647265726672677268726972707271727272737274727572767277727872797280728172827283728472857286728772887289729072917292729372947295729672977298729973007301730273037304730573067307730873097310731173127313731473157316731773187319732073217322732373247325732673277328732973307331733273337334733573367337733873397340734173427343734473457346734773487349735073517352735373547355735673577358735973607361736273637364736573667367736873697370737173727373737473757376737773787379738073817382738373847385738673877388738973907391739273937394739573967397739873997400740174027403740474057406740774087409741074117412741374147415741674177418741974207421742274237424742574267427742874297430743174327433743474357436743774387439744074417442744374447445744674477448744974507451745274537454745574567457745874597460746174627463746474657466746774687469747074717472747374747475747674777478747974807481748274837484748574867487748874897490749174927493749474957496749774987499750075017502750375047505750675077508750975107511751275137514751575167517751875197520752175227523752475257526752775287529753075317532753375347535753675377538753975407541754275437544754575467547754875497550755175527553755475557556755775587559756075617562756375647565756675677568756975707571757275737574757575767577757875797580758175827583758475857586758775887589759075917592759375947595759675977598
  1. /*
  2. * Copyright (C) 2007 Oracle. All rights reserved.
  3. *
  4. * This program is free software; you can redistribute it and/or
  5. * modify it under the terms of the GNU General Public
  6. * License v2 as published by the Free Software Foundation.
  7. *
  8. * This program is distributed in the hope that it will be useful,
  9. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11. * General Public License for more details.
  12. *
  13. * You should have received a copy of the GNU General Public
  14. * License along with this program; if not, write to the
  15. * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  16. * Boston, MA 021110-1307, USA.
  17. */
  18. #include <linux/sched.h>
  19. #include <linux/pagemap.h>
  20. #include <linux/writeback.h>
  21. #include <linux/blkdev.h>
  22. #include <linux/sort.h>
  23. #include <linux/rcupdate.h>
  24. #include <linux/kthread.h>
  25. #include <linux/slab.h>
  26. #include <linux/ratelimit.h>
  27. #include "compat.h"
  28. #include "hash.h"
  29. #include "ctree.h"
  30. #include "disk-io.h"
  31. #include "print-tree.h"
  32. #include "transaction.h"
  33. #include "volumes.h"
  34. #include "locking.h"
  35. #include "free-space-cache.h"
  36. /* control flags for do_chunk_alloc's force field
  37. * CHUNK_ALLOC_NO_FORCE means to only allocate a chunk
  38. * if we really need one.
  39. *
  40. * CHUNK_ALLOC_FORCE means it must try to allocate one
  41. *
  42. * CHUNK_ALLOC_LIMITED means to only try and allocate one
  43. * if we have very few chunks already allocated. This is
  44. * used as part of the clustering code to help make sure
  45. * we have a good pool of storage to cluster in, without
  46. * filling the FS with empty chunks
  47. *
  48. */
  49. enum {
  50. CHUNK_ALLOC_NO_FORCE = 0,
  51. CHUNK_ALLOC_FORCE = 1,
  52. CHUNK_ALLOC_LIMITED = 2,
  53. };
  54. /*
  55. * Control how reservations are dealt with.
  56. *
  57. * RESERVE_FREE - freeing a reservation.
  58. * RESERVE_ALLOC - allocating space and we need to update bytes_may_use for
  59. * ENOSPC accounting
  60. * RESERVE_ALLOC_NO_ACCOUNT - allocating space and we should not update
  61. * bytes_may_use as the ENOSPC accounting is done elsewhere
  62. */
  63. enum {
  64. RESERVE_FREE = 0,
  65. RESERVE_ALLOC = 1,
  66. RESERVE_ALLOC_NO_ACCOUNT = 2,
  67. };
  68. static int update_block_group(struct btrfs_trans_handle *trans,
  69. struct btrfs_root *root,
  70. u64 bytenr, u64 num_bytes, int alloc);
  71. static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
  72. struct btrfs_root *root,
  73. u64 bytenr, u64 num_bytes, u64 parent,
  74. u64 root_objectid, u64 owner_objectid,
  75. u64 owner_offset, int refs_to_drop,
  76. struct btrfs_delayed_extent_op *extra_op);
  77. static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
  78. struct extent_buffer *leaf,
  79. struct btrfs_extent_item *ei);
  80. static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
  81. struct btrfs_root *root,
  82. u64 parent, u64 root_objectid,
  83. u64 flags, u64 owner, u64 offset,
  84. struct btrfs_key *ins, int ref_mod);
  85. static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
  86. struct btrfs_root *root,
  87. u64 parent, u64 root_objectid,
  88. u64 flags, struct btrfs_disk_key *key,
  89. int level, struct btrfs_key *ins);
  90. static int do_chunk_alloc(struct btrfs_trans_handle *trans,
  91. struct btrfs_root *extent_root, u64 alloc_bytes,
  92. u64 flags, int force);
  93. static int find_next_key(struct btrfs_path *path, int level,
  94. struct btrfs_key *key);
  95. static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
  96. int dump_block_groups);
  97. static int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache,
  98. u64 num_bytes, int reserve);
  99. static noinline int
  100. block_group_cache_done(struct btrfs_block_group_cache *cache)
  101. {
  102. smp_mb();
  103. return cache->cached == BTRFS_CACHE_FINISHED;
  104. }
  105. static int block_group_bits(struct btrfs_block_group_cache *cache, u64 bits)
  106. {
  107. return (cache->flags & bits) == bits;
  108. }
  109. static void btrfs_get_block_group(struct btrfs_block_group_cache *cache)
  110. {
  111. atomic_inc(&cache->count);
  112. }
  113. void btrfs_put_block_group(struct btrfs_block_group_cache *cache)
  114. {
  115. if (atomic_dec_and_test(&cache->count)) {
  116. WARN_ON(cache->pinned > 0);
  117. WARN_ON(cache->reserved > 0);
  118. kfree(cache->free_space_ctl);
  119. kfree(cache);
  120. }
  121. }
  122. /*
  123. * this adds the block group to the fs_info rb tree for the block group
  124. * cache
  125. */
  126. static int btrfs_add_block_group_cache(struct btrfs_fs_info *info,
  127. struct btrfs_block_group_cache *block_group)
  128. {
  129. struct rb_node **p;
  130. struct rb_node *parent = NULL;
  131. struct btrfs_block_group_cache *cache;
  132. spin_lock(&info->block_group_cache_lock);
  133. p = &info->block_group_cache_tree.rb_node;
  134. while (*p) {
  135. parent = *p;
  136. cache = rb_entry(parent, struct btrfs_block_group_cache,
  137. cache_node);
  138. if (block_group->key.objectid < cache->key.objectid) {
  139. p = &(*p)->rb_left;
  140. } else if (block_group->key.objectid > cache->key.objectid) {
  141. p = &(*p)->rb_right;
  142. } else {
  143. spin_unlock(&info->block_group_cache_lock);
  144. return -EEXIST;
  145. }
  146. }
  147. rb_link_node(&block_group->cache_node, parent, p);
  148. rb_insert_color(&block_group->cache_node,
  149. &info->block_group_cache_tree);
  150. spin_unlock(&info->block_group_cache_lock);
  151. return 0;
  152. }
  153. /*
  154. * This will return the block group at or after bytenr if contains is 0, else
  155. * it will return the block group that contains the bytenr
  156. */
  157. static struct btrfs_block_group_cache *
  158. block_group_cache_tree_search(struct btrfs_fs_info *info, u64 bytenr,
  159. int contains)
  160. {
  161. struct btrfs_block_group_cache *cache, *ret = NULL;
  162. struct rb_node *n;
  163. u64 end, start;
  164. spin_lock(&info->block_group_cache_lock);
  165. n = info->block_group_cache_tree.rb_node;
  166. while (n) {
  167. cache = rb_entry(n, struct btrfs_block_group_cache,
  168. cache_node);
  169. end = cache->key.objectid + cache->key.offset - 1;
  170. start = cache->key.objectid;
  171. if (bytenr < start) {
  172. if (!contains && (!ret || start < ret->key.objectid))
  173. ret = cache;
  174. n = n->rb_left;
  175. } else if (bytenr > start) {
  176. if (contains && bytenr <= end) {
  177. ret = cache;
  178. break;
  179. }
  180. n = n->rb_right;
  181. } else {
  182. ret = cache;
  183. break;
  184. }
  185. }
  186. if (ret)
  187. btrfs_get_block_group(ret);
  188. spin_unlock(&info->block_group_cache_lock);
  189. return ret;
  190. }
  191. static int add_excluded_extent(struct btrfs_root *root,
  192. u64 start, u64 num_bytes)
  193. {
  194. u64 end = start + num_bytes - 1;
  195. set_extent_bits(&root->fs_info->freed_extents[0],
  196. start, end, EXTENT_UPTODATE, GFP_NOFS);
  197. set_extent_bits(&root->fs_info->freed_extents[1],
  198. start, end, EXTENT_UPTODATE, GFP_NOFS);
  199. return 0;
  200. }
  201. static void free_excluded_extents(struct btrfs_root *root,
  202. struct btrfs_block_group_cache *cache)
  203. {
  204. u64 start, end;
  205. start = cache->key.objectid;
  206. end = start + cache->key.offset - 1;
  207. clear_extent_bits(&root->fs_info->freed_extents[0],
  208. start, end, EXTENT_UPTODATE, GFP_NOFS);
  209. clear_extent_bits(&root->fs_info->freed_extents[1],
  210. start, end, EXTENT_UPTODATE, GFP_NOFS);
  211. }
  212. static int exclude_super_stripes(struct btrfs_root *root,
  213. struct btrfs_block_group_cache *cache)
  214. {
  215. u64 bytenr;
  216. u64 *logical;
  217. int stripe_len;
  218. int i, nr, ret;
  219. if (cache->key.objectid < BTRFS_SUPER_INFO_OFFSET) {
  220. stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->key.objectid;
  221. cache->bytes_super += stripe_len;
  222. ret = add_excluded_extent(root, cache->key.objectid,
  223. stripe_len);
  224. BUG_ON(ret);
  225. }
  226. for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
  227. bytenr = btrfs_sb_offset(i);
  228. ret = btrfs_rmap_block(&root->fs_info->mapping_tree,
  229. cache->key.objectid, bytenr,
  230. 0, &logical, &nr, &stripe_len);
  231. BUG_ON(ret);
  232. while (nr--) {
  233. cache->bytes_super += stripe_len;
  234. ret = add_excluded_extent(root, logical[nr],
  235. stripe_len);
  236. BUG_ON(ret);
  237. }
  238. kfree(logical);
  239. }
  240. return 0;
  241. }
  242. static struct btrfs_caching_control *
  243. get_caching_control(struct btrfs_block_group_cache *cache)
  244. {
  245. struct btrfs_caching_control *ctl;
  246. spin_lock(&cache->lock);
  247. if (cache->cached != BTRFS_CACHE_STARTED) {
  248. spin_unlock(&cache->lock);
  249. return NULL;
  250. }
  251. /* We're loading it the fast way, so we don't have a caching_ctl. */
  252. if (!cache->caching_ctl) {
  253. spin_unlock(&cache->lock);
  254. return NULL;
  255. }
  256. ctl = cache->caching_ctl;
  257. atomic_inc(&ctl->count);
  258. spin_unlock(&cache->lock);
  259. return ctl;
  260. }
  261. static void put_caching_control(struct btrfs_caching_control *ctl)
  262. {
  263. if (atomic_dec_and_test(&ctl->count))
  264. kfree(ctl);
  265. }
  266. /*
  267. * this is only called by cache_block_group, since we could have freed extents
  268. * we need to check the pinned_extents for any extents that can't be used yet
  269. * since their free space will be released as soon as the transaction commits.
  270. */
  271. static u64 add_new_free_space(struct btrfs_block_group_cache *block_group,
  272. struct btrfs_fs_info *info, u64 start, u64 end)
  273. {
  274. u64 extent_start, extent_end, size, total_added = 0;
  275. int ret;
  276. while (start < end) {
  277. ret = find_first_extent_bit(info->pinned_extents, start,
  278. &extent_start, &extent_end,
  279. EXTENT_DIRTY | EXTENT_UPTODATE);
  280. if (ret)
  281. break;
  282. if (extent_start <= start) {
  283. start = extent_end + 1;
  284. } else if (extent_start > start && extent_start < end) {
  285. size = extent_start - start;
  286. total_added += size;
  287. ret = btrfs_add_free_space(block_group, start,
  288. size);
  289. BUG_ON(ret);
  290. start = extent_end + 1;
  291. } else {
  292. break;
  293. }
  294. }
  295. if (start < end) {
  296. size = end - start;
  297. total_added += size;
  298. ret = btrfs_add_free_space(block_group, start, size);
  299. BUG_ON(ret);
  300. }
  301. return total_added;
  302. }
  303. static noinline void caching_thread(struct btrfs_work *work)
  304. {
  305. struct btrfs_block_group_cache *block_group;
  306. struct btrfs_fs_info *fs_info;
  307. struct btrfs_caching_control *caching_ctl;
  308. struct btrfs_root *extent_root;
  309. struct btrfs_path *path;
  310. struct extent_buffer *leaf;
  311. struct btrfs_key key;
  312. u64 total_found = 0;
  313. u64 last = 0;
  314. u32 nritems;
  315. int ret = 0;
  316. caching_ctl = container_of(work, struct btrfs_caching_control, work);
  317. block_group = caching_ctl->block_group;
  318. fs_info = block_group->fs_info;
  319. extent_root = fs_info->extent_root;
  320. path = btrfs_alloc_path();
  321. if (!path)
  322. goto out;
  323. last = max_t(u64, block_group->key.objectid, BTRFS_SUPER_INFO_OFFSET);
  324. /*
  325. * We don't want to deadlock with somebody trying to allocate a new
  326. * extent for the extent root while also trying to search the extent
  327. * root to add free space. So we skip locking and search the commit
  328. * root, since its read-only
  329. */
  330. path->skip_locking = 1;
  331. path->search_commit_root = 1;
  332. path->reada = 1;
  333. key.objectid = last;
  334. key.offset = 0;
  335. key.type = BTRFS_EXTENT_ITEM_KEY;
  336. again:
  337. mutex_lock(&caching_ctl->mutex);
  338. /* need to make sure the commit_root doesn't disappear */
  339. down_read(&fs_info->extent_commit_sem);
  340. ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
  341. if (ret < 0)
  342. goto err;
  343. leaf = path->nodes[0];
  344. nritems = btrfs_header_nritems(leaf);
  345. while (1) {
  346. if (btrfs_fs_closing(fs_info) > 1) {
  347. last = (u64)-1;
  348. break;
  349. }
  350. if (path->slots[0] < nritems) {
  351. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  352. } else {
  353. ret = find_next_key(path, 0, &key);
  354. if (ret)
  355. break;
  356. if (need_resched() ||
  357. btrfs_next_leaf(extent_root, path)) {
  358. caching_ctl->progress = last;
  359. btrfs_release_path(path);
  360. up_read(&fs_info->extent_commit_sem);
  361. mutex_unlock(&caching_ctl->mutex);
  362. cond_resched();
  363. goto again;
  364. }
  365. leaf = path->nodes[0];
  366. nritems = btrfs_header_nritems(leaf);
  367. continue;
  368. }
  369. if (key.objectid < block_group->key.objectid) {
  370. path->slots[0]++;
  371. continue;
  372. }
  373. if (key.objectid >= block_group->key.objectid +
  374. block_group->key.offset)
  375. break;
  376. if (key.type == BTRFS_EXTENT_ITEM_KEY) {
  377. total_found += add_new_free_space(block_group,
  378. fs_info, last,
  379. key.objectid);
  380. last = key.objectid + key.offset;
  381. if (total_found > (1024 * 1024 * 2)) {
  382. total_found = 0;
  383. wake_up(&caching_ctl->wait);
  384. }
  385. }
  386. path->slots[0]++;
  387. }
  388. ret = 0;
  389. total_found += add_new_free_space(block_group, fs_info, last,
  390. block_group->key.objectid +
  391. block_group->key.offset);
  392. caching_ctl->progress = (u64)-1;
  393. spin_lock(&block_group->lock);
  394. block_group->caching_ctl = NULL;
  395. block_group->cached = BTRFS_CACHE_FINISHED;
  396. spin_unlock(&block_group->lock);
  397. err:
  398. btrfs_free_path(path);
  399. up_read(&fs_info->extent_commit_sem);
  400. free_excluded_extents(extent_root, block_group);
  401. mutex_unlock(&caching_ctl->mutex);
  402. out:
  403. wake_up(&caching_ctl->wait);
  404. put_caching_control(caching_ctl);
  405. btrfs_put_block_group(block_group);
  406. }
  407. static int cache_block_group(struct btrfs_block_group_cache *cache,
  408. struct btrfs_trans_handle *trans,
  409. struct btrfs_root *root,
  410. int load_cache_only)
  411. {
  412. struct btrfs_fs_info *fs_info = cache->fs_info;
  413. struct btrfs_caching_control *caching_ctl;
  414. int ret = 0;
  415. smp_mb();
  416. if (cache->cached != BTRFS_CACHE_NO)
  417. return 0;
  418. /*
  419. * We can't do the read from on-disk cache during a commit since we need
  420. * to have the normal tree locking. Also if we are currently trying to
  421. * allocate blocks for the tree root we can't do the fast caching since
  422. * we likely hold important locks.
  423. */
  424. if (trans && (!trans->transaction->in_commit) &&
  425. (root && root != root->fs_info->tree_root) &&
  426. btrfs_test_opt(root, SPACE_CACHE)) {
  427. spin_lock(&cache->lock);
  428. if (cache->cached != BTRFS_CACHE_NO) {
  429. spin_unlock(&cache->lock);
  430. return 0;
  431. }
  432. cache->cached = BTRFS_CACHE_STARTED;
  433. spin_unlock(&cache->lock);
  434. ret = load_free_space_cache(fs_info, cache);
  435. spin_lock(&cache->lock);
  436. if (ret == 1) {
  437. cache->cached = BTRFS_CACHE_FINISHED;
  438. cache->last_byte_to_unpin = (u64)-1;
  439. } else {
  440. cache->cached = BTRFS_CACHE_NO;
  441. }
  442. spin_unlock(&cache->lock);
  443. if (ret == 1) {
  444. free_excluded_extents(fs_info->extent_root, cache);
  445. return 0;
  446. }
  447. }
  448. if (load_cache_only)
  449. return 0;
  450. caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_NOFS);
  451. BUG_ON(!caching_ctl);
  452. INIT_LIST_HEAD(&caching_ctl->list);
  453. mutex_init(&caching_ctl->mutex);
  454. init_waitqueue_head(&caching_ctl->wait);
  455. caching_ctl->block_group = cache;
  456. caching_ctl->progress = cache->key.objectid;
  457. /* one for caching kthread, one for caching block group list */
  458. atomic_set(&caching_ctl->count, 2);
  459. caching_ctl->work.func = caching_thread;
  460. spin_lock(&cache->lock);
  461. if (cache->cached != BTRFS_CACHE_NO) {
  462. spin_unlock(&cache->lock);
  463. kfree(caching_ctl);
  464. return 0;
  465. }
  466. cache->caching_ctl = caching_ctl;
  467. cache->cached = BTRFS_CACHE_STARTED;
  468. spin_unlock(&cache->lock);
  469. down_write(&fs_info->extent_commit_sem);
  470. list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups);
  471. up_write(&fs_info->extent_commit_sem);
  472. btrfs_get_block_group(cache);
  473. btrfs_queue_worker(&fs_info->caching_workers, &caching_ctl->work);
  474. return ret;
  475. }
  476. /*
  477. * return the block group that starts at or after bytenr
  478. */
  479. static struct btrfs_block_group_cache *
  480. btrfs_lookup_first_block_group(struct btrfs_fs_info *info, u64 bytenr)
  481. {
  482. struct btrfs_block_group_cache *cache;
  483. cache = block_group_cache_tree_search(info, bytenr, 0);
  484. return cache;
  485. }
  486. /*
  487. * return the block group that contains the given bytenr
  488. */
  489. struct btrfs_block_group_cache *btrfs_lookup_block_group(
  490. struct btrfs_fs_info *info,
  491. u64 bytenr)
  492. {
  493. struct btrfs_block_group_cache *cache;
  494. cache = block_group_cache_tree_search(info, bytenr, 1);
  495. return cache;
  496. }
  497. static struct btrfs_space_info *__find_space_info(struct btrfs_fs_info *info,
  498. u64 flags)
  499. {
  500. struct list_head *head = &info->space_info;
  501. struct btrfs_space_info *found;
  502. flags &= BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_SYSTEM |
  503. BTRFS_BLOCK_GROUP_METADATA;
  504. rcu_read_lock();
  505. list_for_each_entry_rcu(found, head, list) {
  506. if (found->flags & flags) {
  507. rcu_read_unlock();
  508. return found;
  509. }
  510. }
  511. rcu_read_unlock();
  512. return NULL;
  513. }
  514. /*
  515. * after adding space to the filesystem, we need to clear the full flags
  516. * on all the space infos.
  517. */
  518. void btrfs_clear_space_info_full(struct btrfs_fs_info *info)
  519. {
  520. struct list_head *head = &info->space_info;
  521. struct btrfs_space_info *found;
  522. rcu_read_lock();
  523. list_for_each_entry_rcu(found, head, list)
  524. found->full = 0;
  525. rcu_read_unlock();
  526. }
  527. static u64 div_factor(u64 num, int factor)
  528. {
  529. if (factor == 10)
  530. return num;
  531. num *= factor;
  532. do_div(num, 10);
  533. return num;
  534. }
  535. static u64 div_factor_fine(u64 num, int factor)
  536. {
  537. if (factor == 100)
  538. return num;
  539. num *= factor;
  540. do_div(num, 100);
  541. return num;
  542. }
  543. u64 btrfs_find_block_group(struct btrfs_root *root,
  544. u64 search_start, u64 search_hint, int owner)
  545. {
  546. struct btrfs_block_group_cache *cache;
  547. u64 used;
  548. u64 last = max(search_hint, search_start);
  549. u64 group_start = 0;
  550. int full_search = 0;
  551. int factor = 9;
  552. int wrapped = 0;
  553. again:
  554. while (1) {
  555. cache = btrfs_lookup_first_block_group(root->fs_info, last);
  556. if (!cache)
  557. break;
  558. spin_lock(&cache->lock);
  559. last = cache->key.objectid + cache->key.offset;
  560. used = btrfs_block_group_used(&cache->item);
  561. if ((full_search || !cache->ro) &&
  562. block_group_bits(cache, BTRFS_BLOCK_GROUP_METADATA)) {
  563. if (used + cache->pinned + cache->reserved <
  564. div_factor(cache->key.offset, factor)) {
  565. group_start = cache->key.objectid;
  566. spin_unlock(&cache->lock);
  567. btrfs_put_block_group(cache);
  568. goto found;
  569. }
  570. }
  571. spin_unlock(&cache->lock);
  572. btrfs_put_block_group(cache);
  573. cond_resched();
  574. }
  575. if (!wrapped) {
  576. last = search_start;
  577. wrapped = 1;
  578. goto again;
  579. }
  580. if (!full_search && factor < 10) {
  581. last = search_start;
  582. full_search = 1;
  583. factor = 10;
  584. goto again;
  585. }
  586. found:
  587. return group_start;
  588. }
  589. /* simple helper to search for an existing extent at a given offset */
  590. int btrfs_lookup_extent(struct btrfs_root *root, u64 start, u64 len)
  591. {
  592. int ret;
  593. struct btrfs_key key;
  594. struct btrfs_path *path;
  595. path = btrfs_alloc_path();
  596. if (!path)
  597. return -ENOMEM;
  598. key.objectid = start;
  599. key.offset = len;
  600. btrfs_set_key_type(&key, BTRFS_EXTENT_ITEM_KEY);
  601. ret = btrfs_search_slot(NULL, root->fs_info->extent_root, &key, path,
  602. 0, 0);
  603. btrfs_free_path(path);
  604. return ret;
  605. }
  606. /*
  607. * helper function to lookup reference count and flags of extent.
  608. *
  609. * the head node for delayed ref is used to store the sum of all the
  610. * reference count modifications queued up in the rbtree. the head
  611. * node may also store the extent flags to set. This way you can check
  612. * to see what the reference count and extent flags would be if all of
  613. * the delayed refs are not processed.
  614. */
  615. int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans,
  616. struct btrfs_root *root, u64 bytenr,
  617. u64 num_bytes, u64 *refs, u64 *flags)
  618. {
  619. struct btrfs_delayed_ref_head *head;
  620. struct btrfs_delayed_ref_root *delayed_refs;
  621. struct btrfs_path *path;
  622. struct btrfs_extent_item *ei;
  623. struct extent_buffer *leaf;
  624. struct btrfs_key key;
  625. u32 item_size;
  626. u64 num_refs;
  627. u64 extent_flags;
  628. int ret;
  629. path = btrfs_alloc_path();
  630. if (!path)
  631. return -ENOMEM;
  632. key.objectid = bytenr;
  633. key.type = BTRFS_EXTENT_ITEM_KEY;
  634. key.offset = num_bytes;
  635. if (!trans) {
  636. path->skip_locking = 1;
  637. path->search_commit_root = 1;
  638. }
  639. again:
  640. ret = btrfs_search_slot(trans, root->fs_info->extent_root,
  641. &key, path, 0, 0);
  642. if (ret < 0)
  643. goto out_free;
  644. if (ret == 0) {
  645. leaf = path->nodes[0];
  646. item_size = btrfs_item_size_nr(leaf, path->slots[0]);
  647. if (item_size >= sizeof(*ei)) {
  648. ei = btrfs_item_ptr(leaf, path->slots[0],
  649. struct btrfs_extent_item);
  650. num_refs = btrfs_extent_refs(leaf, ei);
  651. extent_flags = btrfs_extent_flags(leaf, ei);
  652. } else {
  653. #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
  654. struct btrfs_extent_item_v0 *ei0;
  655. BUG_ON(item_size != sizeof(*ei0));
  656. ei0 = btrfs_item_ptr(leaf, path->slots[0],
  657. struct btrfs_extent_item_v0);
  658. num_refs = btrfs_extent_refs_v0(leaf, ei0);
  659. /* FIXME: this isn't correct for data */
  660. extent_flags = BTRFS_BLOCK_FLAG_FULL_BACKREF;
  661. #else
  662. BUG();
  663. #endif
  664. }
  665. BUG_ON(num_refs == 0);
  666. } else {
  667. num_refs = 0;
  668. extent_flags = 0;
  669. ret = 0;
  670. }
  671. if (!trans)
  672. goto out;
  673. delayed_refs = &trans->transaction->delayed_refs;
  674. spin_lock(&delayed_refs->lock);
  675. head = btrfs_find_delayed_ref_head(trans, bytenr);
  676. if (head) {
  677. if (!mutex_trylock(&head->mutex)) {
  678. atomic_inc(&head->node.refs);
  679. spin_unlock(&delayed_refs->lock);
  680. btrfs_release_path(path);
  681. /*
  682. * Mutex was contended, block until it's released and try
  683. * again
  684. */
  685. mutex_lock(&head->mutex);
  686. mutex_unlock(&head->mutex);
  687. btrfs_put_delayed_ref(&head->node);
  688. goto again;
  689. }
  690. if (head->extent_op && head->extent_op->update_flags)
  691. extent_flags |= head->extent_op->flags_to_set;
  692. else
  693. BUG_ON(num_refs == 0);
  694. num_refs += head->node.ref_mod;
  695. mutex_unlock(&head->mutex);
  696. }
  697. spin_unlock(&delayed_refs->lock);
  698. out:
  699. WARN_ON(num_refs == 0);
  700. if (refs)
  701. *refs = num_refs;
  702. if (flags)
  703. *flags = extent_flags;
  704. out_free:
  705. btrfs_free_path(path);
  706. return ret;
  707. }
  708. /*
  709. * Back reference rules. Back refs have three main goals:
  710. *
  711. * 1) differentiate between all holders of references to an extent so that
  712. * when a reference is dropped we can make sure it was a valid reference
  713. * before freeing the extent.
  714. *
  715. * 2) Provide enough information to quickly find the holders of an extent
  716. * if we notice a given block is corrupted or bad.
  717. *
  718. * 3) Make it easy to migrate blocks for FS shrinking or storage pool
  719. * maintenance. This is actually the same as #2, but with a slightly
  720. * different use case.
  721. *
  722. * There are two kinds of back refs. The implicit back refs is optimized
  723. * for pointers in non-shared tree blocks. For a given pointer in a block,
  724. * back refs of this kind provide information about the block's owner tree
  725. * and the pointer's key. These information allow us to find the block by
  726. * b-tree searching. The full back refs is for pointers in tree blocks not
  727. * referenced by their owner trees. The location of tree block is recorded
  728. * in the back refs. Actually the full back refs is generic, and can be
  729. * used in all cases the implicit back refs is used. The major shortcoming
  730. * of the full back refs is its overhead. Every time a tree block gets
  731. * COWed, we have to update back refs entry for all pointers in it.
  732. *
  733. * For a newly allocated tree block, we use implicit back refs for
  734. * pointers in it. This means most tree related operations only involve
  735. * implicit back refs. For a tree block created in old transaction, the
  736. * only way to drop a reference to it is COW it. So we can detect the
  737. * event that tree block loses its owner tree's reference and do the
  738. * back refs conversion.
  739. *
  740. * When a tree block is COW'd through a tree, there are four cases:
  741. *
  742. * The reference count of the block is one and the tree is the block's
  743. * owner tree. Nothing to do in this case.
  744. *
  745. * The reference count of the block is one and the tree is not the
  746. * block's owner tree. In this case, full back refs is used for pointers
  747. * in the block. Remove these full back refs, add implicit back refs for
  748. * every pointers in the new block.
  749. *
  750. * The reference count of the block is greater than one and the tree is
  751. * the block's owner tree. In this case, implicit back refs is used for
  752. * pointers in the block. Add full back refs for every pointers in the
  753. * block, increase lower level extents' reference counts. The original
  754. * implicit back refs are entailed to the new block.
  755. *
  756. * The reference count of the block is greater than one and the tree is
  757. * not the block's owner tree. Add implicit back refs for every pointer in
  758. * the new block, increase lower level extents' reference count.
  759. *
  760. * Back Reference Key composing:
  761. *
  762. * The key objectid corresponds to the first byte in the extent,
  763. * The key type is used to differentiate between types of back refs.
  764. * There are different meanings of the key offset for different types
  765. * of back refs.
  766. *
  767. * File extents can be referenced by:
  768. *
  769. * - multiple snapshots, subvolumes, or different generations in one subvol
  770. * - different files inside a single subvolume
  771. * - different offsets inside a file (bookend extents in file.c)
  772. *
  773. * The extent ref structure for the implicit back refs has fields for:
  774. *
  775. * - Objectid of the subvolume root
  776. * - objectid of the file holding the reference
  777. * - original offset in the file
  778. * - how many bookend extents
  779. *
  780. * The key offset for the implicit back refs is hash of the first
  781. * three fields.
  782. *
  783. * The extent ref structure for the full back refs has field for:
  784. *
  785. * - number of pointers in the tree leaf
  786. *
  787. * The key offset for the implicit back refs is the first byte of
  788. * the tree leaf
  789. *
  790. * When a file extent is allocated, The implicit back refs is used.
  791. * the fields are filled in:
  792. *
  793. * (root_key.objectid, inode objectid, offset in file, 1)
  794. *
  795. * When a file extent is removed file truncation, we find the
  796. * corresponding implicit back refs and check the following fields:
  797. *
  798. * (btrfs_header_owner(leaf), inode objectid, offset in file)
  799. *
  800. * Btree extents can be referenced by:
  801. *
  802. * - Different subvolumes
  803. *
  804. * Both the implicit back refs and the full back refs for tree blocks
  805. * only consist of key. The key offset for the implicit back refs is
  806. * objectid of block's owner tree. The key offset for the full back refs
  807. * is the first byte of parent block.
  808. *
  809. * When implicit back refs is used, information about the lowest key and
  810. * level of the tree block are required. These information are stored in
  811. * tree block info structure.
  812. */
  813. #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
  814. static int convert_extent_item_v0(struct btrfs_trans_handle *trans,
  815. struct btrfs_root *root,
  816. struct btrfs_path *path,
  817. u64 owner, u32 extra_size)
  818. {
  819. struct btrfs_extent_item *item;
  820. struct btrfs_extent_item_v0 *ei0;
  821. struct btrfs_extent_ref_v0 *ref0;
  822. struct btrfs_tree_block_info *bi;
  823. struct extent_buffer *leaf;
  824. struct btrfs_key key;
  825. struct btrfs_key found_key;
  826. u32 new_size = sizeof(*item);
  827. u64 refs;
  828. int ret;
  829. leaf = path->nodes[0];
  830. BUG_ON(btrfs_item_size_nr(leaf, path->slots[0]) != sizeof(*ei0));
  831. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  832. ei0 = btrfs_item_ptr(leaf, path->slots[0],
  833. struct btrfs_extent_item_v0);
  834. refs = btrfs_extent_refs_v0(leaf, ei0);
  835. if (owner == (u64)-1) {
  836. while (1) {
  837. if (path->slots[0] >= btrfs_header_nritems(leaf)) {
  838. ret = btrfs_next_leaf(root, path);
  839. if (ret < 0)
  840. return ret;
  841. BUG_ON(ret > 0);
  842. leaf = path->nodes[0];
  843. }
  844. btrfs_item_key_to_cpu(leaf, &found_key,
  845. path->slots[0]);
  846. BUG_ON(key.objectid != found_key.objectid);
  847. if (found_key.type != BTRFS_EXTENT_REF_V0_KEY) {
  848. path->slots[0]++;
  849. continue;
  850. }
  851. ref0 = btrfs_item_ptr(leaf, path->slots[0],
  852. struct btrfs_extent_ref_v0);
  853. owner = btrfs_ref_objectid_v0(leaf, ref0);
  854. break;
  855. }
  856. }
  857. btrfs_release_path(path);
  858. if (owner < BTRFS_FIRST_FREE_OBJECTID)
  859. new_size += sizeof(*bi);
  860. new_size -= sizeof(*ei0);
  861. ret = btrfs_search_slot(trans, root, &key, path,
  862. new_size + extra_size, 1);
  863. if (ret < 0)
  864. return ret;
  865. BUG_ON(ret);
  866. ret = btrfs_extend_item(trans, root, path, new_size);
  867. leaf = path->nodes[0];
  868. item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
  869. btrfs_set_extent_refs(leaf, item, refs);
  870. /* FIXME: get real generation */
  871. btrfs_set_extent_generation(leaf, item, 0);
  872. if (owner < BTRFS_FIRST_FREE_OBJECTID) {
  873. btrfs_set_extent_flags(leaf, item,
  874. BTRFS_EXTENT_FLAG_TREE_BLOCK |
  875. BTRFS_BLOCK_FLAG_FULL_BACKREF);
  876. bi = (struct btrfs_tree_block_info *)(item + 1);
  877. /* FIXME: get first key of the block */
  878. memset_extent_buffer(leaf, 0, (unsigned long)bi, sizeof(*bi));
  879. btrfs_set_tree_block_level(leaf, bi, (int)owner);
  880. } else {
  881. btrfs_set_extent_flags(leaf, item, BTRFS_EXTENT_FLAG_DATA);
  882. }
  883. btrfs_mark_buffer_dirty(leaf);
  884. return 0;
  885. }
  886. #endif
  887. static u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset)
  888. {
  889. u32 high_crc = ~(u32)0;
  890. u32 low_crc = ~(u32)0;
  891. __le64 lenum;
  892. lenum = cpu_to_le64(root_objectid);
  893. high_crc = crc32c(high_crc, &lenum, sizeof(lenum));
  894. lenum = cpu_to_le64(owner);
  895. low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
  896. lenum = cpu_to_le64(offset);
  897. low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
  898. return ((u64)high_crc << 31) ^ (u64)low_crc;
  899. }
  900. static u64 hash_extent_data_ref_item(struct extent_buffer *leaf,
  901. struct btrfs_extent_data_ref *ref)
  902. {
  903. return hash_extent_data_ref(btrfs_extent_data_ref_root(leaf, ref),
  904. btrfs_extent_data_ref_objectid(leaf, ref),
  905. btrfs_extent_data_ref_offset(leaf, ref));
  906. }
  907. static int match_extent_data_ref(struct extent_buffer *leaf,
  908. struct btrfs_extent_data_ref *ref,
  909. u64 root_objectid, u64 owner, u64 offset)
  910. {
  911. if (btrfs_extent_data_ref_root(leaf, ref) != root_objectid ||
  912. btrfs_extent_data_ref_objectid(leaf, ref) != owner ||
  913. btrfs_extent_data_ref_offset(leaf, ref) != offset)
  914. return 0;
  915. return 1;
  916. }
  917. static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans,
  918. struct btrfs_root *root,
  919. struct btrfs_path *path,
  920. u64 bytenr, u64 parent,
  921. u64 root_objectid,
  922. u64 owner, u64 offset)
  923. {
  924. struct btrfs_key key;
  925. struct btrfs_extent_data_ref *ref;
  926. struct extent_buffer *leaf;
  927. u32 nritems;
  928. int ret;
  929. int recow;
  930. int err = -ENOENT;
  931. key.objectid = bytenr;
  932. if (parent) {
  933. key.type = BTRFS_SHARED_DATA_REF_KEY;
  934. key.offset = parent;
  935. } else {
  936. key.type = BTRFS_EXTENT_DATA_REF_KEY;
  937. key.offset = hash_extent_data_ref(root_objectid,
  938. owner, offset);
  939. }
  940. again:
  941. recow = 0;
  942. ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
  943. if (ret < 0) {
  944. err = ret;
  945. goto fail;
  946. }
  947. if (parent) {
  948. if (!ret)
  949. return 0;
  950. #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
  951. key.type = BTRFS_EXTENT_REF_V0_KEY;
  952. btrfs_release_path(path);
  953. ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
  954. if (ret < 0) {
  955. err = ret;
  956. goto fail;
  957. }
  958. if (!ret)
  959. return 0;
  960. #endif
  961. goto fail;
  962. }
  963. leaf = path->nodes[0];
  964. nritems = btrfs_header_nritems(leaf);
  965. while (1) {
  966. if (path->slots[0] >= nritems) {
  967. ret = btrfs_next_leaf(root, path);
  968. if (ret < 0)
  969. err = ret;
  970. if (ret)
  971. goto fail;
  972. leaf = path->nodes[0];
  973. nritems = btrfs_header_nritems(leaf);
  974. recow = 1;
  975. }
  976. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  977. if (key.objectid != bytenr ||
  978. key.type != BTRFS_EXTENT_DATA_REF_KEY)
  979. goto fail;
  980. ref = btrfs_item_ptr(leaf, path->slots[0],
  981. struct btrfs_extent_data_ref);
  982. if (match_extent_data_ref(leaf, ref, root_objectid,
  983. owner, offset)) {
  984. if (recow) {
  985. btrfs_release_path(path);
  986. goto again;
  987. }
  988. err = 0;
  989. break;
  990. }
  991. path->slots[0]++;
  992. }
  993. fail:
  994. return err;
  995. }
  996. static noinline int insert_extent_data_ref(struct btrfs_trans_handle *trans,
  997. struct btrfs_root *root,
  998. struct btrfs_path *path,
  999. u64 bytenr, u64 parent,
  1000. u64 root_objectid, u64 owner,
  1001. u64 offset, int refs_to_add)
  1002. {
  1003. struct btrfs_key key;
  1004. struct extent_buffer *leaf;
  1005. u32 size;
  1006. u32 num_refs;
  1007. int ret;
  1008. key.objectid = bytenr;
  1009. if (parent) {
  1010. key.type = BTRFS_SHARED_DATA_REF_KEY;
  1011. key.offset = parent;
  1012. size = sizeof(struct btrfs_shared_data_ref);
  1013. } else {
  1014. key.type = BTRFS_EXTENT_DATA_REF_KEY;
  1015. key.offset = hash_extent_data_ref(root_objectid,
  1016. owner, offset);
  1017. size = sizeof(struct btrfs_extent_data_ref);
  1018. }
  1019. ret = btrfs_insert_empty_item(trans, root, path, &key, size);
  1020. if (ret && ret != -EEXIST)
  1021. goto fail;
  1022. leaf = path->nodes[0];
  1023. if (parent) {
  1024. struct btrfs_shared_data_ref *ref;
  1025. ref = btrfs_item_ptr(leaf, path->slots[0],
  1026. struct btrfs_shared_data_ref);
  1027. if (ret == 0) {
  1028. btrfs_set_shared_data_ref_count(leaf, ref, refs_to_add);
  1029. } else {
  1030. num_refs = btrfs_shared_data_ref_count(leaf, ref);
  1031. num_refs += refs_to_add;
  1032. btrfs_set_shared_data_ref_count(leaf, ref, num_refs);
  1033. }
  1034. } else {
  1035. struct btrfs_extent_data_ref *ref;
  1036. while (ret == -EEXIST) {
  1037. ref = btrfs_item_ptr(leaf, path->slots[0],
  1038. struct btrfs_extent_data_ref);
  1039. if (match_extent_data_ref(leaf, ref, root_objectid,
  1040. owner, offset))
  1041. break;
  1042. btrfs_release_path(path);
  1043. key.offset++;
  1044. ret = btrfs_insert_empty_item(trans, root, path, &key,
  1045. size);
  1046. if (ret && ret != -EEXIST)
  1047. goto fail;
  1048. leaf = path->nodes[0];
  1049. }
  1050. ref = btrfs_item_ptr(leaf, path->slots[0],
  1051. struct btrfs_extent_data_ref);
  1052. if (ret == 0) {
  1053. btrfs_set_extent_data_ref_root(leaf, ref,
  1054. root_objectid);
  1055. btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
  1056. btrfs_set_extent_data_ref_offset(leaf, ref, offset);
  1057. btrfs_set_extent_data_ref_count(leaf, ref, refs_to_add);
  1058. } else {
  1059. num_refs = btrfs_extent_data_ref_count(leaf, ref);
  1060. num_refs += refs_to_add;
  1061. btrfs_set_extent_data_ref_count(leaf, ref, num_refs);
  1062. }
  1063. }
  1064. btrfs_mark_buffer_dirty(leaf);
  1065. ret = 0;
  1066. fail:
  1067. btrfs_release_path(path);
  1068. return ret;
  1069. }
  1070. static noinline int remove_extent_data_ref(struct btrfs_trans_handle *trans,
  1071. struct btrfs_root *root,
  1072. struct btrfs_path *path,
  1073. int refs_to_drop)
  1074. {
  1075. struct btrfs_key key;
  1076. struct btrfs_extent_data_ref *ref1 = NULL;
  1077. struct btrfs_shared_data_ref *ref2 = NULL;
  1078. struct extent_buffer *leaf;
  1079. u32 num_refs = 0;
  1080. int ret = 0;
  1081. leaf = path->nodes[0];
  1082. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  1083. if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
  1084. ref1 = btrfs_item_ptr(leaf, path->slots[0],
  1085. struct btrfs_extent_data_ref);
  1086. num_refs = btrfs_extent_data_ref_count(leaf, ref1);
  1087. } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
  1088. ref2 = btrfs_item_ptr(leaf, path->slots[0],
  1089. struct btrfs_shared_data_ref);
  1090. num_refs = btrfs_shared_data_ref_count(leaf, ref2);
  1091. #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
  1092. } else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
  1093. struct btrfs_extent_ref_v0 *ref0;
  1094. ref0 = btrfs_item_ptr(leaf, path->slots[0],
  1095. struct btrfs_extent_ref_v0);
  1096. num_refs = btrfs_ref_count_v0(leaf, ref0);
  1097. #endif
  1098. } else {
  1099. BUG();
  1100. }
  1101. BUG_ON(num_refs < refs_to_drop);
  1102. num_refs -= refs_to_drop;
  1103. if (num_refs == 0) {
  1104. ret = btrfs_del_item(trans, root, path);
  1105. } else {
  1106. if (key.type == BTRFS_EXTENT_DATA_REF_KEY)
  1107. btrfs_set_extent_data_ref_count(leaf, ref1, num_refs);
  1108. else if (key.type == BTRFS_SHARED_DATA_REF_KEY)
  1109. btrfs_set_shared_data_ref_count(leaf, ref2, num_refs);
  1110. #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
  1111. else {
  1112. struct btrfs_extent_ref_v0 *ref0;
  1113. ref0 = btrfs_item_ptr(leaf, path->slots[0],
  1114. struct btrfs_extent_ref_v0);
  1115. btrfs_set_ref_count_v0(leaf, ref0, num_refs);
  1116. }
  1117. #endif
  1118. btrfs_mark_buffer_dirty(leaf);
  1119. }
  1120. return ret;
  1121. }
  1122. static noinline u32 extent_data_ref_count(struct btrfs_root *root,
  1123. struct btrfs_path *path,
  1124. struct btrfs_extent_inline_ref *iref)
  1125. {
  1126. struct btrfs_key key;
  1127. struct extent_buffer *leaf;
  1128. struct btrfs_extent_data_ref *ref1;
  1129. struct btrfs_shared_data_ref *ref2;
  1130. u32 num_refs = 0;
  1131. leaf = path->nodes[0];
  1132. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  1133. if (iref) {
  1134. if (btrfs_extent_inline_ref_type(leaf, iref) ==
  1135. BTRFS_EXTENT_DATA_REF_KEY) {
  1136. ref1 = (struct btrfs_extent_data_ref *)(&iref->offset);
  1137. num_refs = btrfs_extent_data_ref_count(leaf, ref1);
  1138. } else {
  1139. ref2 = (struct btrfs_shared_data_ref *)(iref + 1);
  1140. num_refs = btrfs_shared_data_ref_count(leaf, ref2);
  1141. }
  1142. } else if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
  1143. ref1 = btrfs_item_ptr(leaf, path->slots[0],
  1144. struct btrfs_extent_data_ref);
  1145. num_refs = btrfs_extent_data_ref_count(leaf, ref1);
  1146. } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
  1147. ref2 = btrfs_item_ptr(leaf, path->slots[0],
  1148. struct btrfs_shared_data_ref);
  1149. num_refs = btrfs_shared_data_ref_count(leaf, ref2);
  1150. #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
  1151. } else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
  1152. struct btrfs_extent_ref_v0 *ref0;
  1153. ref0 = btrfs_item_ptr(leaf, path->slots[0],
  1154. struct btrfs_extent_ref_v0);
  1155. num_refs = btrfs_ref_count_v0(leaf, ref0);
  1156. #endif
  1157. } else {
  1158. WARN_ON(1);
  1159. }
  1160. return num_refs;
  1161. }
  1162. static noinline int lookup_tree_block_ref(struct btrfs_trans_handle *trans,
  1163. struct btrfs_root *root,
  1164. struct btrfs_path *path,
  1165. u64 bytenr, u64 parent,
  1166. u64 root_objectid)
  1167. {
  1168. struct btrfs_key key;
  1169. int ret;
  1170. key.objectid = bytenr;
  1171. if (parent) {
  1172. key.type = BTRFS_SHARED_BLOCK_REF_KEY;
  1173. key.offset = parent;
  1174. } else {
  1175. key.type = BTRFS_TREE_BLOCK_REF_KEY;
  1176. key.offset = root_objectid;
  1177. }
  1178. ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
  1179. if (ret > 0)
  1180. ret = -ENOENT;
  1181. #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
  1182. if (ret == -ENOENT && parent) {
  1183. btrfs_release_path(path);
  1184. key.type = BTRFS_EXTENT_REF_V0_KEY;
  1185. ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
  1186. if (ret > 0)
  1187. ret = -ENOENT;
  1188. }
  1189. #endif
  1190. return ret;
  1191. }
  1192. static noinline int insert_tree_block_ref(struct btrfs_trans_handle *trans,
  1193. struct btrfs_root *root,
  1194. struct btrfs_path *path,
  1195. u64 bytenr, u64 parent,
  1196. u64 root_objectid)
  1197. {
  1198. struct btrfs_key key;
  1199. int ret;
  1200. key.objectid = bytenr;
  1201. if (parent) {
  1202. key.type = BTRFS_SHARED_BLOCK_REF_KEY;
  1203. key.offset = parent;
  1204. } else {
  1205. key.type = BTRFS_TREE_BLOCK_REF_KEY;
  1206. key.offset = root_objectid;
  1207. }
  1208. ret = btrfs_insert_empty_item(trans, root, path, &key, 0);
  1209. btrfs_release_path(path);
  1210. return ret;
  1211. }
  1212. static inline int extent_ref_type(u64 parent, u64 owner)
  1213. {
  1214. int type;
  1215. if (owner < BTRFS_FIRST_FREE_OBJECTID) {
  1216. if (parent > 0)
  1217. type = BTRFS_SHARED_BLOCK_REF_KEY;
  1218. else
  1219. type = BTRFS_TREE_BLOCK_REF_KEY;
  1220. } else {
  1221. if (parent > 0)
  1222. type = BTRFS_SHARED_DATA_REF_KEY;
  1223. else
  1224. type = BTRFS_EXTENT_DATA_REF_KEY;
  1225. }
  1226. return type;
  1227. }
  1228. static int find_next_key(struct btrfs_path *path, int level,
  1229. struct btrfs_key *key)
  1230. {
  1231. for (; level < BTRFS_MAX_LEVEL; level++) {
  1232. if (!path->nodes[level])
  1233. break;
  1234. if (path->slots[level] + 1 >=
  1235. btrfs_header_nritems(path->nodes[level]))
  1236. continue;
  1237. if (level == 0)
  1238. btrfs_item_key_to_cpu(path->nodes[level], key,
  1239. path->slots[level] + 1);
  1240. else
  1241. btrfs_node_key_to_cpu(path->nodes[level], key,
  1242. path->slots[level] + 1);
  1243. return 0;
  1244. }
  1245. return 1;
  1246. }
  1247. /*
  1248. * look for inline back ref. if back ref is found, *ref_ret is set
  1249. * to the address of inline back ref, and 0 is returned.
  1250. *
  1251. * if back ref isn't found, *ref_ret is set to the address where it
  1252. * should be inserted, and -ENOENT is returned.
  1253. *
  1254. * if insert is true and there are too many inline back refs, the path
  1255. * points to the extent item, and -EAGAIN is returned.
  1256. *
  1257. * NOTE: inline back refs are ordered in the same way that back ref
  1258. * items in the tree are ordered.
  1259. */
  1260. static noinline_for_stack
  1261. int lookup_inline_extent_backref(struct btrfs_trans_handle *trans,
  1262. struct btrfs_root *root,
  1263. struct btrfs_path *path,
  1264. struct btrfs_extent_inline_ref **ref_ret,
  1265. u64 bytenr, u64 num_bytes,
  1266. u64 parent, u64 root_objectid,
  1267. u64 owner, u64 offset, int insert)
  1268. {
  1269. struct btrfs_key key;
  1270. struct extent_buffer *leaf;
  1271. struct btrfs_extent_item *ei;
  1272. struct btrfs_extent_inline_ref *iref;
  1273. u64 flags;
  1274. u64 item_size;
  1275. unsigned long ptr;
  1276. unsigned long end;
  1277. int extra_size;
  1278. int type;
  1279. int want;
  1280. int ret;
  1281. int err = 0;
  1282. key.objectid = bytenr;
  1283. key.type = BTRFS_EXTENT_ITEM_KEY;
  1284. key.offset = num_bytes;
  1285. want = extent_ref_type(parent, owner);
  1286. if (insert) {
  1287. extra_size = btrfs_extent_inline_ref_size(want);
  1288. path->keep_locks = 1;
  1289. } else
  1290. extra_size = -1;
  1291. ret = btrfs_search_slot(trans, root, &key, path, extra_size, 1);
  1292. if (ret < 0) {
  1293. err = ret;
  1294. goto out;
  1295. }
  1296. BUG_ON(ret);
  1297. leaf = path->nodes[0];
  1298. item_size = btrfs_item_size_nr(leaf, path->slots[0]);
  1299. #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
  1300. if (item_size < sizeof(*ei)) {
  1301. if (!insert) {
  1302. err = -ENOENT;
  1303. goto out;
  1304. }
  1305. ret = convert_extent_item_v0(trans, root, path, owner,
  1306. extra_size);
  1307. if (ret < 0) {
  1308. err = ret;
  1309. goto out;
  1310. }
  1311. leaf = path->nodes[0];
  1312. item_size = btrfs_item_size_nr(leaf, path->slots[0]);
  1313. }
  1314. #endif
  1315. BUG_ON(item_size < sizeof(*ei));
  1316. ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
  1317. flags = btrfs_extent_flags(leaf, ei);
  1318. ptr = (unsigned long)(ei + 1);
  1319. end = (unsigned long)ei + item_size;
  1320. if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
  1321. ptr += sizeof(struct btrfs_tree_block_info);
  1322. BUG_ON(ptr > end);
  1323. } else {
  1324. BUG_ON(!(flags & BTRFS_EXTENT_FLAG_DATA));
  1325. }
  1326. err = -ENOENT;
  1327. while (1) {
  1328. if (ptr >= end) {
  1329. WARN_ON(ptr > end);
  1330. break;
  1331. }
  1332. iref = (struct btrfs_extent_inline_ref *)ptr;
  1333. type = btrfs_extent_inline_ref_type(leaf, iref);
  1334. if (want < type)
  1335. break;
  1336. if (want > type) {
  1337. ptr += btrfs_extent_inline_ref_size(type);
  1338. continue;
  1339. }
  1340. if (type == BTRFS_EXTENT_DATA_REF_KEY) {
  1341. struct btrfs_extent_data_ref *dref;
  1342. dref = (struct btrfs_extent_data_ref *)(&iref->offset);
  1343. if (match_extent_data_ref(leaf, dref, root_objectid,
  1344. owner, offset)) {
  1345. err = 0;
  1346. break;
  1347. }
  1348. if (hash_extent_data_ref_item(leaf, dref) <
  1349. hash_extent_data_ref(root_objectid, owner, offset))
  1350. break;
  1351. } else {
  1352. u64 ref_offset;
  1353. ref_offset = btrfs_extent_inline_ref_offset(leaf, iref);
  1354. if (parent > 0) {
  1355. if (parent == ref_offset) {
  1356. err = 0;
  1357. break;
  1358. }
  1359. if (ref_offset < parent)
  1360. break;
  1361. } else {
  1362. if (root_objectid == ref_offset) {
  1363. err = 0;
  1364. break;
  1365. }
  1366. if (ref_offset < root_objectid)
  1367. break;
  1368. }
  1369. }
  1370. ptr += btrfs_extent_inline_ref_size(type);
  1371. }
  1372. if (err == -ENOENT && insert) {
  1373. if (item_size + extra_size >=
  1374. BTRFS_MAX_EXTENT_ITEM_SIZE(root)) {
  1375. err = -EAGAIN;
  1376. goto out;
  1377. }
  1378. /*
  1379. * To add new inline back ref, we have to make sure
  1380. * there is no corresponding back ref item.
  1381. * For simplicity, we just do not add new inline back
  1382. * ref if there is any kind of item for this block
  1383. */
  1384. if (find_next_key(path, 0, &key) == 0 &&
  1385. key.objectid == bytenr &&
  1386. key.type < BTRFS_BLOCK_GROUP_ITEM_KEY) {
  1387. err = -EAGAIN;
  1388. goto out;
  1389. }
  1390. }
  1391. *ref_ret = (struct btrfs_extent_inline_ref *)ptr;
  1392. out:
  1393. if (insert) {
  1394. path->keep_locks = 0;
  1395. btrfs_unlock_up_safe(path, 1);
  1396. }
  1397. return err;
  1398. }
  1399. /*
  1400. * helper to add new inline back ref
  1401. */
  1402. static noinline_for_stack
  1403. int setup_inline_extent_backref(struct btrfs_trans_handle *trans,
  1404. struct btrfs_root *root,
  1405. struct btrfs_path *path,
  1406. struct btrfs_extent_inline_ref *iref,
  1407. u64 parent, u64 root_objectid,
  1408. u64 owner, u64 offset, int refs_to_add,
  1409. struct btrfs_delayed_extent_op *extent_op)
  1410. {
  1411. struct extent_buffer *leaf;
  1412. struct btrfs_extent_item *ei;
  1413. unsigned long ptr;
  1414. unsigned long end;
  1415. unsigned long item_offset;
  1416. u64 refs;
  1417. int size;
  1418. int type;
  1419. int ret;
  1420. leaf = path->nodes[0];
  1421. ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
  1422. item_offset = (unsigned long)iref - (unsigned long)ei;
  1423. type = extent_ref_type(parent, owner);
  1424. size = btrfs_extent_inline_ref_size(type);
  1425. ret = btrfs_extend_item(trans, root, path, size);
  1426. ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
  1427. refs = btrfs_extent_refs(leaf, ei);
  1428. refs += refs_to_add;
  1429. btrfs_set_extent_refs(leaf, ei, refs);
  1430. if (extent_op)
  1431. __run_delayed_extent_op(extent_op, leaf, ei);
  1432. ptr = (unsigned long)ei + item_offset;
  1433. end = (unsigned long)ei + btrfs_item_size_nr(leaf, path->slots[0]);
  1434. if (ptr < end - size)
  1435. memmove_extent_buffer(leaf, ptr + size, ptr,
  1436. end - size - ptr);
  1437. iref = (struct btrfs_extent_inline_ref *)ptr;
  1438. btrfs_set_extent_inline_ref_type(leaf, iref, type);
  1439. if (type == BTRFS_EXTENT_DATA_REF_KEY) {
  1440. struct btrfs_extent_data_ref *dref;
  1441. dref = (struct btrfs_extent_data_ref *)(&iref->offset);
  1442. btrfs_set_extent_data_ref_root(leaf, dref, root_objectid);
  1443. btrfs_set_extent_data_ref_objectid(leaf, dref, owner);
  1444. btrfs_set_extent_data_ref_offset(leaf, dref, offset);
  1445. btrfs_set_extent_data_ref_count(leaf, dref, refs_to_add);
  1446. } else if (type == BTRFS_SHARED_DATA_REF_KEY) {
  1447. struct btrfs_shared_data_ref *sref;
  1448. sref = (struct btrfs_shared_data_ref *)(iref + 1);
  1449. btrfs_set_shared_data_ref_count(leaf, sref, refs_to_add);
  1450. btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
  1451. } else if (type == BTRFS_SHARED_BLOCK_REF_KEY) {
  1452. btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
  1453. } else {
  1454. btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
  1455. }
  1456. btrfs_mark_buffer_dirty(leaf);
  1457. return 0;
  1458. }
  1459. static int lookup_extent_backref(struct btrfs_trans_handle *trans,
  1460. struct btrfs_root *root,
  1461. struct btrfs_path *path,
  1462. struct btrfs_extent_inline_ref **ref_ret,
  1463. u64 bytenr, u64 num_bytes, u64 parent,
  1464. u64 root_objectid, u64 owner, u64 offset)
  1465. {
  1466. int ret;
  1467. ret = lookup_inline_extent_backref(trans, root, path, ref_ret,
  1468. bytenr, num_bytes, parent,
  1469. root_objectid, owner, offset, 0);
  1470. if (ret != -ENOENT)
  1471. return ret;
  1472. btrfs_release_path(path);
  1473. *ref_ret = NULL;
  1474. if (owner < BTRFS_FIRST_FREE_OBJECTID) {
  1475. ret = lookup_tree_block_ref(trans, root, path, bytenr, parent,
  1476. root_objectid);
  1477. } else {
  1478. ret = lookup_extent_data_ref(trans, root, path, bytenr, parent,
  1479. root_objectid, owner, offset);
  1480. }
  1481. return ret;
  1482. }
  1483. /*
  1484. * helper to update/remove inline back ref
  1485. */
  1486. static noinline_for_stack
  1487. int update_inline_extent_backref(struct btrfs_trans_handle *trans,
  1488. struct btrfs_root *root,
  1489. struct btrfs_path *path,
  1490. struct btrfs_extent_inline_ref *iref,
  1491. int refs_to_mod,
  1492. struct btrfs_delayed_extent_op *extent_op)
  1493. {
  1494. struct extent_buffer *leaf;
  1495. struct btrfs_extent_item *ei;
  1496. struct btrfs_extent_data_ref *dref = NULL;
  1497. struct btrfs_shared_data_ref *sref = NULL;
  1498. unsigned long ptr;
  1499. unsigned long end;
  1500. u32 item_size;
  1501. int size;
  1502. int type;
  1503. int ret;
  1504. u64 refs;
  1505. leaf = path->nodes[0];
  1506. ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
  1507. refs = btrfs_extent_refs(leaf, ei);
  1508. WARN_ON(refs_to_mod < 0 && refs + refs_to_mod <= 0);
  1509. refs += refs_to_mod;
  1510. btrfs_set_extent_refs(leaf, ei, refs);
  1511. if (extent_op)
  1512. __run_delayed_extent_op(extent_op, leaf, ei);
  1513. type = btrfs_extent_inline_ref_type(leaf, iref);
  1514. if (type == BTRFS_EXTENT_DATA_REF_KEY) {
  1515. dref = (struct btrfs_extent_data_ref *)(&iref->offset);
  1516. refs = btrfs_extent_data_ref_count(leaf, dref);
  1517. } else if (type == BTRFS_SHARED_DATA_REF_KEY) {
  1518. sref = (struct btrfs_shared_data_ref *)(iref + 1);
  1519. refs = btrfs_shared_data_ref_count(leaf, sref);
  1520. } else {
  1521. refs = 1;
  1522. BUG_ON(refs_to_mod != -1);
  1523. }
  1524. BUG_ON(refs_to_mod < 0 && refs < -refs_to_mod);
  1525. refs += refs_to_mod;
  1526. if (refs > 0) {
  1527. if (type == BTRFS_EXTENT_DATA_REF_KEY)
  1528. btrfs_set_extent_data_ref_count(leaf, dref, refs);
  1529. else
  1530. btrfs_set_shared_data_ref_count(leaf, sref, refs);
  1531. } else {
  1532. size = btrfs_extent_inline_ref_size(type);
  1533. item_size = btrfs_item_size_nr(leaf, path->slots[0]);
  1534. ptr = (unsigned long)iref;
  1535. end = (unsigned long)ei + item_size;
  1536. if (ptr + size < end)
  1537. memmove_extent_buffer(leaf, ptr, ptr + size,
  1538. end - ptr - size);
  1539. item_size -= size;
  1540. ret = btrfs_truncate_item(trans, root, path, item_size, 1);
  1541. }
  1542. btrfs_mark_buffer_dirty(leaf);
  1543. return 0;
  1544. }
  1545. static noinline_for_stack
  1546. int insert_inline_extent_backref(struct btrfs_trans_handle *trans,
  1547. struct btrfs_root *root,
  1548. struct btrfs_path *path,
  1549. u64 bytenr, u64 num_bytes, u64 parent,
  1550. u64 root_objectid, u64 owner,
  1551. u64 offset, int refs_to_add,
  1552. struct btrfs_delayed_extent_op *extent_op)
  1553. {
  1554. struct btrfs_extent_inline_ref *iref;
  1555. int ret;
  1556. ret = lookup_inline_extent_backref(trans, root, path, &iref,
  1557. bytenr, num_bytes, parent,
  1558. root_objectid, owner, offset, 1);
  1559. if (ret == 0) {
  1560. BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID);
  1561. ret = update_inline_extent_backref(trans, root, path, iref,
  1562. refs_to_add, extent_op);
  1563. } else if (ret == -ENOENT) {
  1564. ret = setup_inline_extent_backref(trans, root, path, iref,
  1565. parent, root_objectid,
  1566. owner, offset, refs_to_add,
  1567. extent_op);
  1568. }
  1569. return ret;
  1570. }
  1571. static int insert_extent_backref(struct btrfs_trans_handle *trans,
  1572. struct btrfs_root *root,
  1573. struct btrfs_path *path,
  1574. u64 bytenr, u64 parent, u64 root_objectid,
  1575. u64 owner, u64 offset, int refs_to_add)
  1576. {
  1577. int ret;
  1578. if (owner < BTRFS_FIRST_FREE_OBJECTID) {
  1579. BUG_ON(refs_to_add != 1);
  1580. ret = insert_tree_block_ref(trans, root, path, bytenr,
  1581. parent, root_objectid);
  1582. } else {
  1583. ret = insert_extent_data_ref(trans, root, path, bytenr,
  1584. parent, root_objectid,
  1585. owner, offset, refs_to_add);
  1586. }
  1587. return ret;
  1588. }
  1589. static int remove_extent_backref(struct btrfs_trans_handle *trans,
  1590. struct btrfs_root *root,
  1591. struct btrfs_path *path,
  1592. struct btrfs_extent_inline_ref *iref,
  1593. int refs_to_drop, int is_data)
  1594. {
  1595. int ret;
  1596. BUG_ON(!is_data && refs_to_drop != 1);
  1597. if (iref) {
  1598. ret = update_inline_extent_backref(trans, root, path, iref,
  1599. -refs_to_drop, NULL);
  1600. } else if (is_data) {
  1601. ret = remove_extent_data_ref(trans, root, path, refs_to_drop);
  1602. } else {
  1603. ret = btrfs_del_item(trans, root, path);
  1604. }
  1605. return ret;
  1606. }
  1607. static int btrfs_issue_discard(struct block_device *bdev,
  1608. u64 start, u64 len)
  1609. {
  1610. return blkdev_issue_discard(bdev, start >> 9, len >> 9, GFP_NOFS, 0);
  1611. }
  1612. static int btrfs_discard_extent(struct btrfs_root *root, u64 bytenr,
  1613. u64 num_bytes, u64 *actual_bytes)
  1614. {
  1615. int ret;
  1616. u64 discarded_bytes = 0;
  1617. struct btrfs_multi_bio *multi = NULL;
  1618. /* Tell the block device(s) that the sectors can be discarded */
  1619. ret = btrfs_map_block(&root->fs_info->mapping_tree, REQ_DISCARD,
  1620. bytenr, &num_bytes, &multi, 0);
  1621. if (!ret) {
  1622. struct btrfs_bio_stripe *stripe = multi->stripes;
  1623. int i;
  1624. for (i = 0; i < multi->num_stripes; i++, stripe++) {
  1625. if (!stripe->dev->can_discard)
  1626. continue;
  1627. ret = btrfs_issue_discard(stripe->dev->bdev,
  1628. stripe->physical,
  1629. stripe->length);
  1630. if (!ret)
  1631. discarded_bytes += stripe->length;
  1632. else if (ret != -EOPNOTSUPP)
  1633. break;
  1634. /*
  1635. * Just in case we get back EOPNOTSUPP for some reason,
  1636. * just ignore the return value so we don't screw up
  1637. * people calling discard_extent.
  1638. */
  1639. ret = 0;
  1640. }
  1641. kfree(multi);
  1642. }
  1643. if (actual_bytes)
  1644. *actual_bytes = discarded_bytes;
  1645. return ret;
  1646. }
  1647. int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
  1648. struct btrfs_root *root,
  1649. u64 bytenr, u64 num_bytes, u64 parent,
  1650. u64 root_objectid, u64 owner, u64 offset)
  1651. {
  1652. int ret;
  1653. BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID &&
  1654. root_objectid == BTRFS_TREE_LOG_OBJECTID);
  1655. if (owner < BTRFS_FIRST_FREE_OBJECTID) {
  1656. ret = btrfs_add_delayed_tree_ref(trans, bytenr, num_bytes,
  1657. parent, root_objectid, (int)owner,
  1658. BTRFS_ADD_DELAYED_REF, NULL);
  1659. } else {
  1660. ret = btrfs_add_delayed_data_ref(trans, bytenr, num_bytes,
  1661. parent, root_objectid, owner, offset,
  1662. BTRFS_ADD_DELAYED_REF, NULL);
  1663. }
  1664. return ret;
  1665. }
  1666. static int __btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
  1667. struct btrfs_root *root,
  1668. u64 bytenr, u64 num_bytes,
  1669. u64 parent, u64 root_objectid,
  1670. u64 owner, u64 offset, int refs_to_add,
  1671. struct btrfs_delayed_extent_op *extent_op)
  1672. {
  1673. struct btrfs_path *path;
  1674. struct extent_buffer *leaf;
  1675. struct btrfs_extent_item *item;
  1676. u64 refs;
  1677. int ret;
  1678. int err = 0;
  1679. path = btrfs_alloc_path();
  1680. if (!path)
  1681. return -ENOMEM;
  1682. path->reada = 1;
  1683. path->leave_spinning = 1;
  1684. /* this will setup the path even if it fails to insert the back ref */
  1685. ret = insert_inline_extent_backref(trans, root->fs_info->extent_root,
  1686. path, bytenr, num_bytes, parent,
  1687. root_objectid, owner, offset,
  1688. refs_to_add, extent_op);
  1689. if (ret == 0)
  1690. goto out;
  1691. if (ret != -EAGAIN) {
  1692. err = ret;
  1693. goto out;
  1694. }
  1695. leaf = path->nodes[0];
  1696. item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
  1697. refs = btrfs_extent_refs(leaf, item);
  1698. btrfs_set_extent_refs(leaf, item, refs + refs_to_add);
  1699. if (extent_op)
  1700. __run_delayed_extent_op(extent_op, leaf, item);
  1701. btrfs_mark_buffer_dirty(leaf);
  1702. btrfs_release_path(path);
  1703. path->reada = 1;
  1704. path->leave_spinning = 1;
  1705. /* now insert the actual backref */
  1706. ret = insert_extent_backref(trans, root->fs_info->extent_root,
  1707. path, bytenr, parent, root_objectid,
  1708. owner, offset, refs_to_add);
  1709. BUG_ON(ret);
  1710. out:
  1711. btrfs_free_path(path);
  1712. return err;
  1713. }
  1714. static int run_delayed_data_ref(struct btrfs_trans_handle *trans,
  1715. struct btrfs_root *root,
  1716. struct btrfs_delayed_ref_node *node,
  1717. struct btrfs_delayed_extent_op *extent_op,
  1718. int insert_reserved)
  1719. {
  1720. int ret = 0;
  1721. struct btrfs_delayed_data_ref *ref;
  1722. struct btrfs_key ins;
  1723. u64 parent = 0;
  1724. u64 ref_root = 0;
  1725. u64 flags = 0;
  1726. ins.objectid = node->bytenr;
  1727. ins.offset = node->num_bytes;
  1728. ins.type = BTRFS_EXTENT_ITEM_KEY;
  1729. ref = btrfs_delayed_node_to_data_ref(node);
  1730. if (node->type == BTRFS_SHARED_DATA_REF_KEY)
  1731. parent = ref->parent;
  1732. else
  1733. ref_root = ref->root;
  1734. if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
  1735. if (extent_op) {
  1736. BUG_ON(extent_op->update_key);
  1737. flags |= extent_op->flags_to_set;
  1738. }
  1739. ret = alloc_reserved_file_extent(trans, root,
  1740. parent, ref_root, flags,
  1741. ref->objectid, ref->offset,
  1742. &ins, node->ref_mod);
  1743. } else if (node->action == BTRFS_ADD_DELAYED_REF) {
  1744. ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
  1745. node->num_bytes, parent,
  1746. ref_root, ref->objectid,
  1747. ref->offset, node->ref_mod,
  1748. extent_op);
  1749. } else if (node->action == BTRFS_DROP_DELAYED_REF) {
  1750. ret = __btrfs_free_extent(trans, root, node->bytenr,
  1751. node->num_bytes, parent,
  1752. ref_root, ref->objectid,
  1753. ref->offset, node->ref_mod,
  1754. extent_op);
  1755. } else {
  1756. BUG();
  1757. }
  1758. return ret;
  1759. }
  1760. static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
  1761. struct extent_buffer *leaf,
  1762. struct btrfs_extent_item *ei)
  1763. {
  1764. u64 flags = btrfs_extent_flags(leaf, ei);
  1765. if (extent_op->update_flags) {
  1766. flags |= extent_op->flags_to_set;
  1767. btrfs_set_extent_flags(leaf, ei, flags);
  1768. }
  1769. if (extent_op->update_key) {
  1770. struct btrfs_tree_block_info *bi;
  1771. BUG_ON(!(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK));
  1772. bi = (struct btrfs_tree_block_info *)(ei + 1);
  1773. btrfs_set_tree_block_key(leaf, bi, &extent_op->key);
  1774. }
  1775. }
  1776. static int run_delayed_extent_op(struct btrfs_trans_handle *trans,
  1777. struct btrfs_root *root,
  1778. struct btrfs_delayed_ref_node *node,
  1779. struct btrfs_delayed_extent_op *extent_op)
  1780. {
  1781. struct btrfs_key key;
  1782. struct btrfs_path *path;
  1783. struct btrfs_extent_item *ei;
  1784. struct extent_buffer *leaf;
  1785. u32 item_size;
  1786. int ret;
  1787. int err = 0;
  1788. path = btrfs_alloc_path();
  1789. if (!path)
  1790. return -ENOMEM;
  1791. key.objectid = node->bytenr;
  1792. key.type = BTRFS_EXTENT_ITEM_KEY;
  1793. key.offset = node->num_bytes;
  1794. path->reada = 1;
  1795. path->leave_spinning = 1;
  1796. ret = btrfs_search_slot(trans, root->fs_info->extent_root, &key,
  1797. path, 0, 1);
  1798. if (ret < 0) {
  1799. err = ret;
  1800. goto out;
  1801. }
  1802. if (ret > 0) {
  1803. err = -EIO;
  1804. goto out;
  1805. }
  1806. leaf = path->nodes[0];
  1807. item_size = btrfs_item_size_nr(leaf, path->slots[0]);
  1808. #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
  1809. if (item_size < sizeof(*ei)) {
  1810. ret = convert_extent_item_v0(trans, root->fs_info->extent_root,
  1811. path, (u64)-1, 0);
  1812. if (ret < 0) {
  1813. err = ret;
  1814. goto out;
  1815. }
  1816. leaf = path->nodes[0];
  1817. item_size = btrfs_item_size_nr(leaf, path->slots[0]);
  1818. }
  1819. #endif
  1820. BUG_ON(item_size < sizeof(*ei));
  1821. ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
  1822. __run_delayed_extent_op(extent_op, leaf, ei);
  1823. btrfs_mark_buffer_dirty(leaf);
  1824. out:
  1825. btrfs_free_path(path);
  1826. return err;
  1827. }
  1828. static int run_delayed_tree_ref(struct btrfs_trans_handle *trans,
  1829. struct btrfs_root *root,
  1830. struct btrfs_delayed_ref_node *node,
  1831. struct btrfs_delayed_extent_op *extent_op,
  1832. int insert_reserved)
  1833. {
  1834. int ret = 0;
  1835. struct btrfs_delayed_tree_ref *ref;
  1836. struct btrfs_key ins;
  1837. u64 parent = 0;
  1838. u64 ref_root = 0;
  1839. ins.objectid = node->bytenr;
  1840. ins.offset = node->num_bytes;
  1841. ins.type = BTRFS_EXTENT_ITEM_KEY;
  1842. ref = btrfs_delayed_node_to_tree_ref(node);
  1843. if (node->type == BTRFS_SHARED_BLOCK_REF_KEY)
  1844. parent = ref->parent;
  1845. else
  1846. ref_root = ref->root;
  1847. BUG_ON(node->ref_mod != 1);
  1848. if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
  1849. BUG_ON(!extent_op || !extent_op->update_flags ||
  1850. !extent_op->update_key);
  1851. ret = alloc_reserved_tree_block(trans, root,
  1852. parent, ref_root,
  1853. extent_op->flags_to_set,
  1854. &extent_op->key,
  1855. ref->level, &ins);
  1856. } else if (node->action == BTRFS_ADD_DELAYED_REF) {
  1857. ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
  1858. node->num_bytes, parent, ref_root,
  1859. ref->level, 0, 1, extent_op);
  1860. } else if (node->action == BTRFS_DROP_DELAYED_REF) {
  1861. ret = __btrfs_free_extent(trans, root, node->bytenr,
  1862. node->num_bytes, parent, ref_root,
  1863. ref->level, 0, 1, extent_op);
  1864. } else {
  1865. BUG();
  1866. }
  1867. return ret;
  1868. }
  1869. /* helper function to actually process a single delayed ref entry */
  1870. static int run_one_delayed_ref(struct btrfs_trans_handle *trans,
  1871. struct btrfs_root *root,
  1872. struct btrfs_delayed_ref_node *node,
  1873. struct btrfs_delayed_extent_op *extent_op,
  1874. int insert_reserved)
  1875. {
  1876. int ret;
  1877. if (btrfs_delayed_ref_is_head(node)) {
  1878. struct btrfs_delayed_ref_head *head;
  1879. /*
  1880. * we've hit the end of the chain and we were supposed
  1881. * to insert this extent into the tree. But, it got
  1882. * deleted before we ever needed to insert it, so all
  1883. * we have to do is clean up the accounting
  1884. */
  1885. BUG_ON(extent_op);
  1886. head = btrfs_delayed_node_to_head(node);
  1887. if (insert_reserved) {
  1888. btrfs_pin_extent(root, node->bytenr,
  1889. node->num_bytes, 1);
  1890. if (head->is_data) {
  1891. ret = btrfs_del_csums(trans, root,
  1892. node->bytenr,
  1893. node->num_bytes);
  1894. BUG_ON(ret);
  1895. }
  1896. }
  1897. mutex_unlock(&head->mutex);
  1898. return 0;
  1899. }
  1900. if (node->type == BTRFS_TREE_BLOCK_REF_KEY ||
  1901. node->type == BTRFS_SHARED_BLOCK_REF_KEY)
  1902. ret = run_delayed_tree_ref(trans, root, node, extent_op,
  1903. insert_reserved);
  1904. else if (node->type == BTRFS_EXTENT_DATA_REF_KEY ||
  1905. node->type == BTRFS_SHARED_DATA_REF_KEY)
  1906. ret = run_delayed_data_ref(trans, root, node, extent_op,
  1907. insert_reserved);
  1908. else
  1909. BUG();
  1910. return ret;
  1911. }
  1912. static noinline struct btrfs_delayed_ref_node *
  1913. select_delayed_ref(struct btrfs_delayed_ref_head *head)
  1914. {
  1915. struct rb_node *node;
  1916. struct btrfs_delayed_ref_node *ref;
  1917. int action = BTRFS_ADD_DELAYED_REF;
  1918. again:
  1919. /*
  1920. * select delayed ref of type BTRFS_ADD_DELAYED_REF first.
  1921. * this prevents ref count from going down to zero when
  1922. * there still are pending delayed ref.
  1923. */
  1924. node = rb_prev(&head->node.rb_node);
  1925. while (1) {
  1926. if (!node)
  1927. break;
  1928. ref = rb_entry(node, struct btrfs_delayed_ref_node,
  1929. rb_node);
  1930. if (ref->bytenr != head->node.bytenr)
  1931. break;
  1932. if (ref->action == action)
  1933. return ref;
  1934. node = rb_prev(node);
  1935. }
  1936. if (action == BTRFS_ADD_DELAYED_REF) {
  1937. action = BTRFS_DROP_DELAYED_REF;
  1938. goto again;
  1939. }
  1940. return NULL;
  1941. }
  1942. static noinline int run_clustered_refs(struct btrfs_trans_handle *trans,
  1943. struct btrfs_root *root,
  1944. struct list_head *cluster)
  1945. {
  1946. struct btrfs_delayed_ref_root *delayed_refs;
  1947. struct btrfs_delayed_ref_node *ref;
  1948. struct btrfs_delayed_ref_head *locked_ref = NULL;
  1949. struct btrfs_delayed_extent_op *extent_op;
  1950. int ret;
  1951. int count = 0;
  1952. int must_insert_reserved = 0;
  1953. delayed_refs = &trans->transaction->delayed_refs;
  1954. while (1) {
  1955. if (!locked_ref) {
  1956. /* pick a new head ref from the cluster list */
  1957. if (list_empty(cluster))
  1958. break;
  1959. locked_ref = list_entry(cluster->next,
  1960. struct btrfs_delayed_ref_head, cluster);
  1961. /* grab the lock that says we are going to process
  1962. * all the refs for this head */
  1963. ret = btrfs_delayed_ref_lock(trans, locked_ref);
  1964. /*
  1965. * we may have dropped the spin lock to get the head
  1966. * mutex lock, and that might have given someone else
  1967. * time to free the head. If that's true, it has been
  1968. * removed from our list and we can move on.
  1969. */
  1970. if (ret == -EAGAIN) {
  1971. locked_ref = NULL;
  1972. count++;
  1973. continue;
  1974. }
  1975. }
  1976. /*
  1977. * record the must insert reserved flag before we
  1978. * drop the spin lock.
  1979. */
  1980. must_insert_reserved = locked_ref->must_insert_reserved;
  1981. locked_ref->must_insert_reserved = 0;
  1982. extent_op = locked_ref->extent_op;
  1983. locked_ref->extent_op = NULL;
  1984. /*
  1985. * locked_ref is the head node, so we have to go one
  1986. * node back for any delayed ref updates
  1987. */
  1988. ref = select_delayed_ref(locked_ref);
  1989. if (!ref) {
  1990. /* All delayed refs have been processed, Go ahead
  1991. * and send the head node to run_one_delayed_ref,
  1992. * so that any accounting fixes can happen
  1993. */
  1994. ref = &locked_ref->node;
  1995. if (extent_op && must_insert_reserved) {
  1996. kfree(extent_op);
  1997. extent_op = NULL;
  1998. }
  1999. if (extent_op) {
  2000. spin_unlock(&delayed_refs->lock);
  2001. ret = run_delayed_extent_op(trans, root,
  2002. ref, extent_op);
  2003. BUG_ON(ret);
  2004. kfree(extent_op);
  2005. cond_resched();
  2006. spin_lock(&delayed_refs->lock);
  2007. continue;
  2008. }
  2009. list_del_init(&locked_ref->cluster);
  2010. locked_ref = NULL;
  2011. }
  2012. ref->in_tree = 0;
  2013. rb_erase(&ref->rb_node, &delayed_refs->root);
  2014. delayed_refs->num_entries--;
  2015. spin_unlock(&delayed_refs->lock);
  2016. ret = run_one_delayed_ref(trans, root, ref, extent_op,
  2017. must_insert_reserved);
  2018. BUG_ON(ret);
  2019. btrfs_put_delayed_ref(ref);
  2020. kfree(extent_op);
  2021. count++;
  2022. cond_resched();
  2023. spin_lock(&delayed_refs->lock);
  2024. }
  2025. return count;
  2026. }
  2027. /*
  2028. * this starts processing the delayed reference count updates and
  2029. * extent insertions we have queued up so far. count can be
  2030. * 0, which means to process everything in the tree at the start
  2031. * of the run (but not newly added entries), or it can be some target
  2032. * number you'd like to process.
  2033. */
  2034. int btrfs_run_delayed_refs(struct btrfs_trans_handle *trans,
  2035. struct btrfs_root *root, unsigned long count)
  2036. {
  2037. struct rb_node *node;
  2038. struct btrfs_delayed_ref_root *delayed_refs;
  2039. struct btrfs_delayed_ref_node *ref;
  2040. struct list_head cluster;
  2041. int ret;
  2042. int run_all = count == (unsigned long)-1;
  2043. int run_most = 0;
  2044. if (root == root->fs_info->extent_root)
  2045. root = root->fs_info->tree_root;
  2046. delayed_refs = &trans->transaction->delayed_refs;
  2047. INIT_LIST_HEAD(&cluster);
  2048. again:
  2049. spin_lock(&delayed_refs->lock);
  2050. if (count == 0) {
  2051. count = delayed_refs->num_entries * 2;
  2052. run_most = 1;
  2053. }
  2054. while (1) {
  2055. if (!(run_all || run_most) &&
  2056. delayed_refs->num_heads_ready < 64)
  2057. break;
  2058. /*
  2059. * go find something we can process in the rbtree. We start at
  2060. * the beginning of the tree, and then build a cluster
  2061. * of refs to process starting at the first one we are able to
  2062. * lock
  2063. */
  2064. ret = btrfs_find_ref_cluster(trans, &cluster,
  2065. delayed_refs->run_delayed_start);
  2066. if (ret)
  2067. break;
  2068. ret = run_clustered_refs(trans, root, &cluster);
  2069. BUG_ON(ret < 0);
  2070. count -= min_t(unsigned long, ret, count);
  2071. if (count == 0)
  2072. break;
  2073. }
  2074. if (run_all) {
  2075. node = rb_first(&delayed_refs->root);
  2076. if (!node)
  2077. goto out;
  2078. count = (unsigned long)-1;
  2079. while (node) {
  2080. ref = rb_entry(node, struct btrfs_delayed_ref_node,
  2081. rb_node);
  2082. if (btrfs_delayed_ref_is_head(ref)) {
  2083. struct btrfs_delayed_ref_head *head;
  2084. head = btrfs_delayed_node_to_head(ref);
  2085. atomic_inc(&ref->refs);
  2086. spin_unlock(&delayed_refs->lock);
  2087. /*
  2088. * Mutex was contended, block until it's
  2089. * released and try again
  2090. */
  2091. mutex_lock(&head->mutex);
  2092. mutex_unlock(&head->mutex);
  2093. btrfs_put_delayed_ref(ref);
  2094. cond_resched();
  2095. goto again;
  2096. }
  2097. node = rb_next(node);
  2098. }
  2099. spin_unlock(&delayed_refs->lock);
  2100. schedule_timeout(1);
  2101. goto again;
  2102. }
  2103. out:
  2104. spin_unlock(&delayed_refs->lock);
  2105. return 0;
  2106. }
  2107. int btrfs_set_disk_extent_flags(struct btrfs_trans_handle *trans,
  2108. struct btrfs_root *root,
  2109. u64 bytenr, u64 num_bytes, u64 flags,
  2110. int is_data)
  2111. {
  2112. struct btrfs_delayed_extent_op *extent_op;
  2113. int ret;
  2114. extent_op = kmalloc(sizeof(*extent_op), GFP_NOFS);
  2115. if (!extent_op)
  2116. return -ENOMEM;
  2117. extent_op->flags_to_set = flags;
  2118. extent_op->update_flags = 1;
  2119. extent_op->update_key = 0;
  2120. extent_op->is_data = is_data ? 1 : 0;
  2121. ret = btrfs_add_delayed_extent_op(trans, bytenr, num_bytes, extent_op);
  2122. if (ret)
  2123. kfree(extent_op);
  2124. return ret;
  2125. }
  2126. static noinline int check_delayed_ref(struct btrfs_trans_handle *trans,
  2127. struct btrfs_root *root,
  2128. struct btrfs_path *path,
  2129. u64 objectid, u64 offset, u64 bytenr)
  2130. {
  2131. struct btrfs_delayed_ref_head *head;
  2132. struct btrfs_delayed_ref_node *ref;
  2133. struct btrfs_delayed_data_ref *data_ref;
  2134. struct btrfs_delayed_ref_root *delayed_refs;
  2135. struct rb_node *node;
  2136. int ret = 0;
  2137. ret = -ENOENT;
  2138. delayed_refs = &trans->transaction->delayed_refs;
  2139. spin_lock(&delayed_refs->lock);
  2140. head = btrfs_find_delayed_ref_head(trans, bytenr);
  2141. if (!head)
  2142. goto out;
  2143. if (!mutex_trylock(&head->mutex)) {
  2144. atomic_inc(&head->node.refs);
  2145. spin_unlock(&delayed_refs->lock);
  2146. btrfs_release_path(path);
  2147. /*
  2148. * Mutex was contended, block until it's released and let
  2149. * caller try again
  2150. */
  2151. mutex_lock(&head->mutex);
  2152. mutex_unlock(&head->mutex);
  2153. btrfs_put_delayed_ref(&head->node);
  2154. return -EAGAIN;
  2155. }
  2156. node = rb_prev(&head->node.rb_node);
  2157. if (!node)
  2158. goto out_unlock;
  2159. ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);
  2160. if (ref->bytenr != bytenr)
  2161. goto out_unlock;
  2162. ret = 1;
  2163. if (ref->type != BTRFS_EXTENT_DATA_REF_KEY)
  2164. goto out_unlock;
  2165. data_ref = btrfs_delayed_node_to_data_ref(ref);
  2166. node = rb_prev(node);
  2167. if (node) {
  2168. ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);
  2169. if (ref->bytenr == bytenr)
  2170. goto out_unlock;
  2171. }
  2172. if (data_ref->root != root->root_key.objectid ||
  2173. data_ref->objectid != objectid || data_ref->offset != offset)
  2174. goto out_unlock;
  2175. ret = 0;
  2176. out_unlock:
  2177. mutex_unlock(&head->mutex);
  2178. out:
  2179. spin_unlock(&delayed_refs->lock);
  2180. return ret;
  2181. }
  2182. static noinline int check_committed_ref(struct btrfs_trans_handle *trans,
  2183. struct btrfs_root *root,
  2184. struct btrfs_path *path,
  2185. u64 objectid, u64 offset, u64 bytenr)
  2186. {
  2187. struct btrfs_root *extent_root = root->fs_info->extent_root;
  2188. struct extent_buffer *leaf;
  2189. struct btrfs_extent_data_ref *ref;
  2190. struct btrfs_extent_inline_ref *iref;
  2191. struct btrfs_extent_item *ei;
  2192. struct btrfs_key key;
  2193. u32 item_size;
  2194. int ret;
  2195. key.objectid = bytenr;
  2196. key.offset = (u64)-1;
  2197. key.type = BTRFS_EXTENT_ITEM_KEY;
  2198. ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
  2199. if (ret < 0)
  2200. goto out;
  2201. BUG_ON(ret == 0);
  2202. ret = -ENOENT;
  2203. if (path->slots[0] == 0)
  2204. goto out;
  2205. path->slots[0]--;
  2206. leaf = path->nodes[0];
  2207. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  2208. if (key.objectid != bytenr || key.type != BTRFS_EXTENT_ITEM_KEY)
  2209. goto out;
  2210. ret = 1;
  2211. item_size = btrfs_item_size_nr(leaf, path->slots[0]);
  2212. #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
  2213. if (item_size < sizeof(*ei)) {
  2214. WARN_ON(item_size != sizeof(struct btrfs_extent_item_v0));
  2215. goto out;
  2216. }
  2217. #endif
  2218. ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
  2219. if (item_size != sizeof(*ei) +
  2220. btrfs_extent_inline_ref_size(BTRFS_EXTENT_DATA_REF_KEY))
  2221. goto out;
  2222. if (btrfs_extent_generation(leaf, ei) <=
  2223. btrfs_root_last_snapshot(&root->root_item))
  2224. goto out;
  2225. iref = (struct btrfs_extent_inline_ref *)(ei + 1);
  2226. if (btrfs_extent_inline_ref_type(leaf, iref) !=
  2227. BTRFS_EXTENT_DATA_REF_KEY)
  2228. goto out;
  2229. ref = (struct btrfs_extent_data_ref *)(&iref->offset);
  2230. if (btrfs_extent_refs(leaf, ei) !=
  2231. btrfs_extent_data_ref_count(leaf, ref) ||
  2232. btrfs_extent_data_ref_root(leaf, ref) !=
  2233. root->root_key.objectid ||
  2234. btrfs_extent_data_ref_objectid(leaf, ref) != objectid ||
  2235. btrfs_extent_data_ref_offset(leaf, ref) != offset)
  2236. goto out;
  2237. ret = 0;
  2238. out:
  2239. return ret;
  2240. }
  2241. int btrfs_cross_ref_exist(struct btrfs_trans_handle *trans,
  2242. struct btrfs_root *root,
  2243. u64 objectid, u64 offset, u64 bytenr)
  2244. {
  2245. struct btrfs_path *path;
  2246. int ret;
  2247. int ret2;
  2248. path = btrfs_alloc_path();
  2249. if (!path)
  2250. return -ENOENT;
  2251. do {
  2252. ret = check_committed_ref(trans, root, path, objectid,
  2253. offset, bytenr);
  2254. if (ret && ret != -ENOENT)
  2255. goto out;
  2256. ret2 = check_delayed_ref(trans, root, path, objectid,
  2257. offset, bytenr);
  2258. } while (ret2 == -EAGAIN);
  2259. if (ret2 && ret2 != -ENOENT) {
  2260. ret = ret2;
  2261. goto out;
  2262. }
  2263. if (ret != -ENOENT || ret2 != -ENOENT)
  2264. ret = 0;
  2265. out:
  2266. btrfs_free_path(path);
  2267. if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
  2268. WARN_ON(ret > 0);
  2269. return ret;
  2270. }
  2271. static int __btrfs_mod_ref(struct btrfs_trans_handle *trans,
  2272. struct btrfs_root *root,
  2273. struct extent_buffer *buf,
  2274. int full_backref, int inc)
  2275. {
  2276. u64 bytenr;
  2277. u64 num_bytes;
  2278. u64 parent;
  2279. u64 ref_root;
  2280. u32 nritems;
  2281. struct btrfs_key key;
  2282. struct btrfs_file_extent_item *fi;
  2283. int i;
  2284. int level;
  2285. int ret = 0;
  2286. int (*process_func)(struct btrfs_trans_handle *, struct btrfs_root *,
  2287. u64, u64, u64, u64, u64, u64);
  2288. ref_root = btrfs_header_owner(buf);
  2289. nritems = btrfs_header_nritems(buf);
  2290. level = btrfs_header_level(buf);
  2291. if (!root->ref_cows && level == 0)
  2292. return 0;
  2293. if (inc)
  2294. process_func = btrfs_inc_extent_ref;
  2295. else
  2296. process_func = btrfs_free_extent;
  2297. if (full_backref)
  2298. parent = buf->start;
  2299. else
  2300. parent = 0;
  2301. for (i = 0; i < nritems; i++) {
  2302. if (level == 0) {
  2303. btrfs_item_key_to_cpu(buf, &key, i);
  2304. if (btrfs_key_type(&key) != BTRFS_EXTENT_DATA_KEY)
  2305. continue;
  2306. fi = btrfs_item_ptr(buf, i,
  2307. struct btrfs_file_extent_item);
  2308. if (btrfs_file_extent_type(buf, fi) ==
  2309. BTRFS_FILE_EXTENT_INLINE)
  2310. continue;
  2311. bytenr = btrfs_file_extent_disk_bytenr(buf, fi);
  2312. if (bytenr == 0)
  2313. continue;
  2314. num_bytes = btrfs_file_extent_disk_num_bytes(buf, fi);
  2315. key.offset -= btrfs_file_extent_offset(buf, fi);
  2316. ret = process_func(trans, root, bytenr, num_bytes,
  2317. parent, ref_root, key.objectid,
  2318. key.offset);
  2319. if (ret)
  2320. goto fail;
  2321. } else {
  2322. bytenr = btrfs_node_blockptr(buf, i);
  2323. num_bytes = btrfs_level_size(root, level - 1);
  2324. ret = process_func(trans, root, bytenr, num_bytes,
  2325. parent, ref_root, level - 1, 0);
  2326. if (ret)
  2327. goto fail;
  2328. }
  2329. }
  2330. return 0;
  2331. fail:
  2332. BUG();
  2333. return ret;
  2334. }
  2335. int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
  2336. struct extent_buffer *buf, int full_backref)
  2337. {
  2338. return __btrfs_mod_ref(trans, root, buf, full_backref, 1);
  2339. }
  2340. int btrfs_dec_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
  2341. struct extent_buffer *buf, int full_backref)
  2342. {
  2343. return __btrfs_mod_ref(trans, root, buf, full_backref, 0);
  2344. }
  2345. static int write_one_cache_group(struct btrfs_trans_handle *trans,
  2346. struct btrfs_root *root,
  2347. struct btrfs_path *path,
  2348. struct btrfs_block_group_cache *cache)
  2349. {
  2350. int ret;
  2351. struct btrfs_root *extent_root = root->fs_info->extent_root;
  2352. unsigned long bi;
  2353. struct extent_buffer *leaf;
  2354. ret = btrfs_search_slot(trans, extent_root, &cache->key, path, 0, 1);
  2355. if (ret < 0)
  2356. goto fail;
  2357. BUG_ON(ret);
  2358. leaf = path->nodes[0];
  2359. bi = btrfs_item_ptr_offset(leaf, path->slots[0]);
  2360. write_extent_buffer(leaf, &cache->item, bi, sizeof(cache->item));
  2361. btrfs_mark_buffer_dirty(leaf);
  2362. btrfs_release_path(path);
  2363. fail:
  2364. if (ret)
  2365. return ret;
  2366. return 0;
  2367. }
  2368. static struct btrfs_block_group_cache *
  2369. next_block_group(struct btrfs_root *root,
  2370. struct btrfs_block_group_cache *cache)
  2371. {
  2372. struct rb_node *node;
  2373. spin_lock(&root->fs_info->block_group_cache_lock);
  2374. node = rb_next(&cache->cache_node);
  2375. btrfs_put_block_group(cache);
  2376. if (node) {
  2377. cache = rb_entry(node, struct btrfs_block_group_cache,
  2378. cache_node);
  2379. btrfs_get_block_group(cache);
  2380. } else
  2381. cache = NULL;
  2382. spin_unlock(&root->fs_info->block_group_cache_lock);
  2383. return cache;
  2384. }
  2385. static int cache_save_setup(struct btrfs_block_group_cache *block_group,
  2386. struct btrfs_trans_handle *trans,
  2387. struct btrfs_path *path)
  2388. {
  2389. struct btrfs_root *root = block_group->fs_info->tree_root;
  2390. struct inode *inode = NULL;
  2391. u64 alloc_hint = 0;
  2392. int dcs = BTRFS_DC_ERROR;
  2393. int num_pages = 0;
  2394. int retries = 0;
  2395. int ret = 0;
  2396. /*
  2397. * If this block group is smaller than 100 megs don't bother caching the
  2398. * block group.
  2399. */
  2400. if (block_group->key.offset < (100 * 1024 * 1024)) {
  2401. spin_lock(&block_group->lock);
  2402. block_group->disk_cache_state = BTRFS_DC_WRITTEN;
  2403. spin_unlock(&block_group->lock);
  2404. return 0;
  2405. }
  2406. again:
  2407. inode = lookup_free_space_inode(root, block_group, path);
  2408. if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) {
  2409. ret = PTR_ERR(inode);
  2410. btrfs_release_path(path);
  2411. goto out;
  2412. }
  2413. if (IS_ERR(inode)) {
  2414. BUG_ON(retries);
  2415. retries++;
  2416. if (block_group->ro)
  2417. goto out_free;
  2418. ret = create_free_space_inode(root, trans, block_group, path);
  2419. if (ret)
  2420. goto out_free;
  2421. goto again;
  2422. }
  2423. /* We've already setup this transaction, go ahead and exit */
  2424. if (block_group->cache_generation == trans->transid &&
  2425. i_size_read(inode)) {
  2426. dcs = BTRFS_DC_SETUP;
  2427. goto out_put;
  2428. }
  2429. /*
  2430. * We want to set the generation to 0, that way if anything goes wrong
  2431. * from here on out we know not to trust this cache when we load up next
  2432. * time.
  2433. */
  2434. BTRFS_I(inode)->generation = 0;
  2435. ret = btrfs_update_inode(trans, root, inode);
  2436. WARN_ON(ret);
  2437. if (i_size_read(inode) > 0) {
  2438. ret = btrfs_truncate_free_space_cache(root, trans, path,
  2439. inode);
  2440. if (ret)
  2441. goto out_put;
  2442. }
  2443. spin_lock(&block_group->lock);
  2444. if (block_group->cached != BTRFS_CACHE_FINISHED) {
  2445. /* We're not cached, don't bother trying to write stuff out */
  2446. dcs = BTRFS_DC_WRITTEN;
  2447. spin_unlock(&block_group->lock);
  2448. goto out_put;
  2449. }
  2450. spin_unlock(&block_group->lock);
  2451. num_pages = (int)div64_u64(block_group->key.offset, 1024 * 1024 * 1024);
  2452. if (!num_pages)
  2453. num_pages = 1;
  2454. /*
  2455. * Just to make absolutely sure we have enough space, we're going to
  2456. * preallocate 12 pages worth of space for each block group. In
  2457. * practice we ought to use at most 8, but we need extra space so we can
  2458. * add our header and have a terminator between the extents and the
  2459. * bitmaps.
  2460. */
  2461. num_pages *= 16;
  2462. num_pages *= PAGE_CACHE_SIZE;
  2463. ret = btrfs_check_data_free_space(inode, num_pages);
  2464. if (ret)
  2465. goto out_put;
  2466. ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, num_pages,
  2467. num_pages, num_pages,
  2468. &alloc_hint);
  2469. if (!ret)
  2470. dcs = BTRFS_DC_SETUP;
  2471. btrfs_free_reserved_data_space(inode, num_pages);
  2472. out_put:
  2473. iput(inode);
  2474. out_free:
  2475. btrfs_release_path(path);
  2476. out:
  2477. spin_lock(&block_group->lock);
  2478. if (!ret)
  2479. block_group->cache_generation = trans->transid;
  2480. block_group->disk_cache_state = dcs;
  2481. spin_unlock(&block_group->lock);
  2482. return ret;
  2483. }
  2484. int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans,
  2485. struct btrfs_root *root)
  2486. {
  2487. struct btrfs_block_group_cache *cache;
  2488. int err = 0;
  2489. struct btrfs_path *path;
  2490. u64 last = 0;
  2491. path = btrfs_alloc_path();
  2492. if (!path)
  2493. return -ENOMEM;
  2494. again:
  2495. while (1) {
  2496. cache = btrfs_lookup_first_block_group(root->fs_info, last);
  2497. while (cache) {
  2498. if (cache->disk_cache_state == BTRFS_DC_CLEAR)
  2499. break;
  2500. cache = next_block_group(root, cache);
  2501. }
  2502. if (!cache) {
  2503. if (last == 0)
  2504. break;
  2505. last = 0;
  2506. continue;
  2507. }
  2508. err = cache_save_setup(cache, trans, path);
  2509. last = cache->key.objectid + cache->key.offset;
  2510. btrfs_put_block_group(cache);
  2511. }
  2512. while (1) {
  2513. if (last == 0) {
  2514. err = btrfs_run_delayed_refs(trans, root,
  2515. (unsigned long)-1);
  2516. BUG_ON(err);
  2517. }
  2518. cache = btrfs_lookup_first_block_group(root->fs_info, last);
  2519. while (cache) {
  2520. if (cache->disk_cache_state == BTRFS_DC_CLEAR) {
  2521. btrfs_put_block_group(cache);
  2522. goto again;
  2523. }
  2524. if (cache->dirty)
  2525. break;
  2526. cache = next_block_group(root, cache);
  2527. }
  2528. if (!cache) {
  2529. if (last == 0)
  2530. break;
  2531. last = 0;
  2532. continue;
  2533. }
  2534. if (cache->disk_cache_state == BTRFS_DC_SETUP)
  2535. cache->disk_cache_state = BTRFS_DC_NEED_WRITE;
  2536. cache->dirty = 0;
  2537. last = cache->key.objectid + cache->key.offset;
  2538. err = write_one_cache_group(trans, root, path, cache);
  2539. BUG_ON(err);
  2540. btrfs_put_block_group(cache);
  2541. }
  2542. while (1) {
  2543. /*
  2544. * I don't think this is needed since we're just marking our
  2545. * preallocated extent as written, but just in case it can't
  2546. * hurt.
  2547. */
  2548. if (last == 0) {
  2549. err = btrfs_run_delayed_refs(trans, root,
  2550. (unsigned long)-1);
  2551. BUG_ON(err);
  2552. }
  2553. cache = btrfs_lookup_first_block_group(root->fs_info, last);
  2554. while (cache) {
  2555. /*
  2556. * Really this shouldn't happen, but it could if we
  2557. * couldn't write the entire preallocated extent and
  2558. * splitting the extent resulted in a new block.
  2559. */
  2560. if (cache->dirty) {
  2561. btrfs_put_block_group(cache);
  2562. goto again;
  2563. }
  2564. if (cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
  2565. break;
  2566. cache = next_block_group(root, cache);
  2567. }
  2568. if (!cache) {
  2569. if (last == 0)
  2570. break;
  2571. last = 0;
  2572. continue;
  2573. }
  2574. btrfs_write_out_cache(root, trans, cache, path);
  2575. /*
  2576. * If we didn't have an error then the cache state is still
  2577. * NEED_WRITE, so we can set it to WRITTEN.
  2578. */
  2579. if (cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
  2580. cache->disk_cache_state = BTRFS_DC_WRITTEN;
  2581. last = cache->key.objectid + cache->key.offset;
  2582. btrfs_put_block_group(cache);
  2583. }
  2584. btrfs_free_path(path);
  2585. return 0;
  2586. }
  2587. int btrfs_extent_readonly(struct btrfs_root *root, u64 bytenr)
  2588. {
  2589. struct btrfs_block_group_cache *block_group;
  2590. int readonly = 0;
  2591. block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
  2592. if (!block_group || block_group->ro)
  2593. readonly = 1;
  2594. if (block_group)
  2595. btrfs_put_block_group(block_group);
  2596. return readonly;
  2597. }
  2598. static int update_space_info(struct btrfs_fs_info *info, u64 flags,
  2599. u64 total_bytes, u64 bytes_used,
  2600. struct btrfs_space_info **space_info)
  2601. {
  2602. struct btrfs_space_info *found;
  2603. int i;
  2604. int factor;
  2605. if (flags & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1 |
  2606. BTRFS_BLOCK_GROUP_RAID10))
  2607. factor = 2;
  2608. else
  2609. factor = 1;
  2610. found = __find_space_info(info, flags);
  2611. if (found) {
  2612. spin_lock(&found->lock);
  2613. found->total_bytes += total_bytes;
  2614. found->disk_total += total_bytes * factor;
  2615. found->bytes_used += bytes_used;
  2616. found->disk_used += bytes_used * factor;
  2617. found->full = 0;
  2618. spin_unlock(&found->lock);
  2619. *space_info = found;
  2620. return 0;
  2621. }
  2622. found = kzalloc(sizeof(*found), GFP_NOFS);
  2623. if (!found)
  2624. return -ENOMEM;
  2625. for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
  2626. INIT_LIST_HEAD(&found->block_groups[i]);
  2627. init_rwsem(&found->groups_sem);
  2628. spin_lock_init(&found->lock);
  2629. found->flags = flags & (BTRFS_BLOCK_GROUP_DATA |
  2630. BTRFS_BLOCK_GROUP_SYSTEM |
  2631. BTRFS_BLOCK_GROUP_METADATA);
  2632. found->total_bytes = total_bytes;
  2633. found->disk_total = total_bytes * factor;
  2634. found->bytes_used = bytes_used;
  2635. found->disk_used = bytes_used * factor;
  2636. found->bytes_pinned = 0;
  2637. found->bytes_reserved = 0;
  2638. found->bytes_readonly = 0;
  2639. found->bytes_may_use = 0;
  2640. found->full = 0;
  2641. found->force_alloc = CHUNK_ALLOC_NO_FORCE;
  2642. found->chunk_alloc = 0;
  2643. found->flush = 0;
  2644. init_waitqueue_head(&found->wait);
  2645. *space_info = found;
  2646. list_add_rcu(&found->list, &info->space_info);
  2647. return 0;
  2648. }
  2649. static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
  2650. {
  2651. u64 extra_flags = flags & (BTRFS_BLOCK_GROUP_RAID0 |
  2652. BTRFS_BLOCK_GROUP_RAID1 |
  2653. BTRFS_BLOCK_GROUP_RAID10 |
  2654. BTRFS_BLOCK_GROUP_DUP);
  2655. if (extra_flags) {
  2656. if (flags & BTRFS_BLOCK_GROUP_DATA)
  2657. fs_info->avail_data_alloc_bits |= extra_flags;
  2658. if (flags & BTRFS_BLOCK_GROUP_METADATA)
  2659. fs_info->avail_metadata_alloc_bits |= extra_flags;
  2660. if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
  2661. fs_info->avail_system_alloc_bits |= extra_flags;
  2662. }
  2663. }
  2664. u64 btrfs_reduce_alloc_profile(struct btrfs_root *root, u64 flags)
  2665. {
  2666. /*
  2667. * we add in the count of missing devices because we want
  2668. * to make sure that any RAID levels on a degraded FS
  2669. * continue to be honored.
  2670. */
  2671. u64 num_devices = root->fs_info->fs_devices->rw_devices +
  2672. root->fs_info->fs_devices->missing_devices;
  2673. if (num_devices == 1)
  2674. flags &= ~(BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID0);
  2675. if (num_devices < 4)
  2676. flags &= ~BTRFS_BLOCK_GROUP_RAID10;
  2677. if ((flags & BTRFS_BLOCK_GROUP_DUP) &&
  2678. (flags & (BTRFS_BLOCK_GROUP_RAID1 |
  2679. BTRFS_BLOCK_GROUP_RAID10))) {
  2680. flags &= ~BTRFS_BLOCK_GROUP_DUP;
  2681. }
  2682. if ((flags & BTRFS_BLOCK_GROUP_RAID1) &&
  2683. (flags & BTRFS_BLOCK_GROUP_RAID10)) {
  2684. flags &= ~BTRFS_BLOCK_GROUP_RAID1;
  2685. }
  2686. if ((flags & BTRFS_BLOCK_GROUP_RAID0) &&
  2687. ((flags & BTRFS_BLOCK_GROUP_RAID1) |
  2688. (flags & BTRFS_BLOCK_GROUP_RAID10) |
  2689. (flags & BTRFS_BLOCK_GROUP_DUP)))
  2690. flags &= ~BTRFS_BLOCK_GROUP_RAID0;
  2691. return flags;
  2692. }
  2693. static u64 get_alloc_profile(struct btrfs_root *root, u64 flags)
  2694. {
  2695. if (flags & BTRFS_BLOCK_GROUP_DATA)
  2696. flags |= root->fs_info->avail_data_alloc_bits &
  2697. root->fs_info->data_alloc_profile;
  2698. else if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
  2699. flags |= root->fs_info->avail_system_alloc_bits &
  2700. root->fs_info->system_alloc_profile;
  2701. else if (flags & BTRFS_BLOCK_GROUP_METADATA)
  2702. flags |= root->fs_info->avail_metadata_alloc_bits &
  2703. root->fs_info->metadata_alloc_profile;
  2704. return btrfs_reduce_alloc_profile(root, flags);
  2705. }
  2706. u64 btrfs_get_alloc_profile(struct btrfs_root *root, int data)
  2707. {
  2708. u64 flags;
  2709. if (data)
  2710. flags = BTRFS_BLOCK_GROUP_DATA;
  2711. else if (root == root->fs_info->chunk_root)
  2712. flags = BTRFS_BLOCK_GROUP_SYSTEM;
  2713. else
  2714. flags = BTRFS_BLOCK_GROUP_METADATA;
  2715. return get_alloc_profile(root, flags);
  2716. }
  2717. void btrfs_set_inode_space_info(struct btrfs_root *root, struct inode *inode)
  2718. {
  2719. BTRFS_I(inode)->space_info = __find_space_info(root->fs_info,
  2720. BTRFS_BLOCK_GROUP_DATA);
  2721. }
  2722. /*
  2723. * This will check the space that the inode allocates from to make sure we have
  2724. * enough space for bytes.
  2725. */
  2726. int btrfs_check_data_free_space(struct inode *inode, u64 bytes)
  2727. {
  2728. struct btrfs_space_info *data_sinfo;
  2729. struct btrfs_root *root = BTRFS_I(inode)->root;
  2730. u64 used;
  2731. int ret = 0, committed = 0, alloc_chunk = 1;
  2732. /* make sure bytes are sectorsize aligned */
  2733. bytes = (bytes + root->sectorsize - 1) & ~((u64)root->sectorsize - 1);
  2734. if (root == root->fs_info->tree_root ||
  2735. BTRFS_I(inode)->location.objectid == BTRFS_FREE_INO_OBJECTID) {
  2736. alloc_chunk = 0;
  2737. committed = 1;
  2738. }
  2739. data_sinfo = BTRFS_I(inode)->space_info;
  2740. if (!data_sinfo)
  2741. goto alloc;
  2742. again:
  2743. /* make sure we have enough space to handle the data first */
  2744. spin_lock(&data_sinfo->lock);
  2745. used = data_sinfo->bytes_used + data_sinfo->bytes_reserved +
  2746. data_sinfo->bytes_pinned + data_sinfo->bytes_readonly +
  2747. data_sinfo->bytes_may_use;
  2748. if (used + bytes > data_sinfo->total_bytes) {
  2749. struct btrfs_trans_handle *trans;
  2750. /*
  2751. * if we don't have enough free bytes in this space then we need
  2752. * to alloc a new chunk.
  2753. */
  2754. if (!data_sinfo->full && alloc_chunk) {
  2755. u64 alloc_target;
  2756. data_sinfo->force_alloc = CHUNK_ALLOC_FORCE;
  2757. spin_unlock(&data_sinfo->lock);
  2758. alloc:
  2759. alloc_target = btrfs_get_alloc_profile(root, 1);
  2760. trans = btrfs_join_transaction(root);
  2761. if (IS_ERR(trans))
  2762. return PTR_ERR(trans);
  2763. ret = do_chunk_alloc(trans, root->fs_info->extent_root,
  2764. bytes + 2 * 1024 * 1024,
  2765. alloc_target,
  2766. CHUNK_ALLOC_NO_FORCE);
  2767. btrfs_end_transaction(trans, root);
  2768. if (ret < 0) {
  2769. if (ret != -ENOSPC)
  2770. return ret;
  2771. else
  2772. goto commit_trans;
  2773. }
  2774. if (!data_sinfo) {
  2775. btrfs_set_inode_space_info(root, inode);
  2776. data_sinfo = BTRFS_I(inode)->space_info;
  2777. }
  2778. goto again;
  2779. }
  2780. /*
  2781. * If we have less pinned bytes than we want to allocate then
  2782. * don't bother committing the transaction, it won't help us.
  2783. */
  2784. if (data_sinfo->bytes_pinned < bytes)
  2785. committed = 1;
  2786. spin_unlock(&data_sinfo->lock);
  2787. /* commit the current transaction and try again */
  2788. commit_trans:
  2789. if (!committed &&
  2790. !atomic_read(&root->fs_info->open_ioctl_trans)) {
  2791. committed = 1;
  2792. trans = btrfs_join_transaction(root);
  2793. if (IS_ERR(trans))
  2794. return PTR_ERR(trans);
  2795. ret = btrfs_commit_transaction(trans, root);
  2796. if (ret)
  2797. return ret;
  2798. goto again;
  2799. }
  2800. return -ENOSPC;
  2801. }
  2802. data_sinfo->bytes_may_use += bytes;
  2803. spin_unlock(&data_sinfo->lock);
  2804. return 0;
  2805. }
  2806. /*
  2807. * Called if we need to clear a data reservation for this inode.
  2808. */
  2809. void btrfs_free_reserved_data_space(struct inode *inode, u64 bytes)
  2810. {
  2811. struct btrfs_root *root = BTRFS_I(inode)->root;
  2812. struct btrfs_space_info *data_sinfo;
  2813. /* make sure bytes are sectorsize aligned */
  2814. bytes = (bytes + root->sectorsize - 1) & ~((u64)root->sectorsize - 1);
  2815. data_sinfo = BTRFS_I(inode)->space_info;
  2816. spin_lock(&data_sinfo->lock);
  2817. data_sinfo->bytes_may_use -= bytes;
  2818. spin_unlock(&data_sinfo->lock);
  2819. }
  2820. static void force_metadata_allocation(struct btrfs_fs_info *info)
  2821. {
  2822. struct list_head *head = &info->space_info;
  2823. struct btrfs_space_info *found;
  2824. rcu_read_lock();
  2825. list_for_each_entry_rcu(found, head, list) {
  2826. if (found->flags & BTRFS_BLOCK_GROUP_METADATA)
  2827. found->force_alloc = CHUNK_ALLOC_FORCE;
  2828. }
  2829. rcu_read_unlock();
  2830. }
  2831. static int should_alloc_chunk(struct btrfs_root *root,
  2832. struct btrfs_space_info *sinfo, u64 alloc_bytes,
  2833. int force)
  2834. {
  2835. struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
  2836. u64 num_bytes = sinfo->total_bytes - sinfo->bytes_readonly;
  2837. u64 num_allocated = sinfo->bytes_used + sinfo->bytes_reserved;
  2838. u64 thresh;
  2839. if (force == CHUNK_ALLOC_FORCE)
  2840. return 1;
  2841. /*
  2842. * We need to take into account the global rsv because for all intents
  2843. * and purposes it's used space. Don't worry about locking the
  2844. * global_rsv, it doesn't change except when the transaction commits.
  2845. */
  2846. num_allocated += global_rsv->size;
  2847. /*
  2848. * in limited mode, we want to have some free space up to
  2849. * about 1% of the FS size.
  2850. */
  2851. if (force == CHUNK_ALLOC_LIMITED) {
  2852. thresh = btrfs_super_total_bytes(root->fs_info->super_copy);
  2853. thresh = max_t(u64, 64 * 1024 * 1024,
  2854. div_factor_fine(thresh, 1));
  2855. if (num_bytes - num_allocated < thresh)
  2856. return 1;
  2857. }
  2858. /*
  2859. * we have two similar checks here, one based on percentage
  2860. * and once based on a hard number of 256MB. The idea
  2861. * is that if we have a good amount of free
  2862. * room, don't allocate a chunk. A good mount is
  2863. * less than 80% utilized of the chunks we have allocated,
  2864. * or more than 256MB free
  2865. */
  2866. if (num_allocated + alloc_bytes + 256 * 1024 * 1024 < num_bytes)
  2867. return 0;
  2868. if (num_allocated + alloc_bytes < div_factor(num_bytes, 8))
  2869. return 0;
  2870. thresh = btrfs_super_total_bytes(root->fs_info->super_copy);
  2871. /* 256MB or 5% of the FS */
  2872. thresh = max_t(u64, 256 * 1024 * 1024, div_factor_fine(thresh, 5));
  2873. if (num_bytes > thresh && sinfo->bytes_used < div_factor(num_bytes, 3))
  2874. return 0;
  2875. return 1;
  2876. }
  2877. static int do_chunk_alloc(struct btrfs_trans_handle *trans,
  2878. struct btrfs_root *extent_root, u64 alloc_bytes,
  2879. u64 flags, int force)
  2880. {
  2881. struct btrfs_space_info *space_info;
  2882. struct btrfs_fs_info *fs_info = extent_root->fs_info;
  2883. int wait_for_alloc = 0;
  2884. int ret = 0;
  2885. flags = btrfs_reduce_alloc_profile(extent_root, flags);
  2886. space_info = __find_space_info(extent_root->fs_info, flags);
  2887. if (!space_info) {
  2888. ret = update_space_info(extent_root->fs_info, flags,
  2889. 0, 0, &space_info);
  2890. BUG_ON(ret);
  2891. }
  2892. BUG_ON(!space_info);
  2893. again:
  2894. spin_lock(&space_info->lock);
  2895. if (space_info->force_alloc)
  2896. force = space_info->force_alloc;
  2897. if (space_info->full) {
  2898. spin_unlock(&space_info->lock);
  2899. return 0;
  2900. }
  2901. if (!should_alloc_chunk(extent_root, space_info, alloc_bytes, force)) {
  2902. spin_unlock(&space_info->lock);
  2903. return 0;
  2904. } else if (space_info->chunk_alloc) {
  2905. wait_for_alloc = 1;
  2906. } else {
  2907. space_info->chunk_alloc = 1;
  2908. }
  2909. spin_unlock(&space_info->lock);
  2910. mutex_lock(&fs_info->chunk_mutex);
  2911. /*
  2912. * The chunk_mutex is held throughout the entirety of a chunk
  2913. * allocation, so once we've acquired the chunk_mutex we know that the
  2914. * other guy is done and we need to recheck and see if we should
  2915. * allocate.
  2916. */
  2917. if (wait_for_alloc) {
  2918. mutex_unlock(&fs_info->chunk_mutex);
  2919. wait_for_alloc = 0;
  2920. goto again;
  2921. }
  2922. /*
  2923. * If we have mixed data/metadata chunks we want to make sure we keep
  2924. * allocating mixed chunks instead of individual chunks.
  2925. */
  2926. if (btrfs_mixed_space_info(space_info))
  2927. flags |= (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA);
  2928. /*
  2929. * if we're doing a data chunk, go ahead and make sure that
  2930. * we keep a reasonable number of metadata chunks allocated in the
  2931. * FS as well.
  2932. */
  2933. if (flags & BTRFS_BLOCK_GROUP_DATA && fs_info->metadata_ratio) {
  2934. fs_info->data_chunk_allocations++;
  2935. if (!(fs_info->data_chunk_allocations %
  2936. fs_info->metadata_ratio))
  2937. force_metadata_allocation(fs_info);
  2938. }
  2939. ret = btrfs_alloc_chunk(trans, extent_root, flags);
  2940. if (ret < 0 && ret != -ENOSPC)
  2941. goto out;
  2942. spin_lock(&space_info->lock);
  2943. if (ret)
  2944. space_info->full = 1;
  2945. else
  2946. ret = 1;
  2947. space_info->force_alloc = CHUNK_ALLOC_NO_FORCE;
  2948. space_info->chunk_alloc = 0;
  2949. spin_unlock(&space_info->lock);
  2950. out:
  2951. mutex_unlock(&extent_root->fs_info->chunk_mutex);
  2952. return ret;
  2953. }
  2954. /*
  2955. * shrink metadata reservation for delalloc
  2956. */
  2957. static int shrink_delalloc(struct btrfs_root *root, u64 to_reclaim,
  2958. bool wait_ordered)
  2959. {
  2960. struct btrfs_block_rsv *block_rsv;
  2961. struct btrfs_space_info *space_info;
  2962. struct btrfs_trans_handle *trans;
  2963. u64 reserved;
  2964. u64 max_reclaim;
  2965. u64 reclaimed = 0;
  2966. long time_left;
  2967. unsigned long nr_pages = (2 * 1024 * 1024) >> PAGE_CACHE_SHIFT;
  2968. int loops = 0;
  2969. unsigned long progress;
  2970. trans = (struct btrfs_trans_handle *)current->journal_info;
  2971. block_rsv = &root->fs_info->delalloc_block_rsv;
  2972. space_info = block_rsv->space_info;
  2973. smp_mb();
  2974. reserved = space_info->bytes_may_use;
  2975. progress = space_info->reservation_progress;
  2976. if (reserved == 0)
  2977. return 0;
  2978. smp_mb();
  2979. if (root->fs_info->delalloc_bytes == 0) {
  2980. if (trans)
  2981. return 0;
  2982. btrfs_wait_ordered_extents(root, 0, 0);
  2983. return 0;
  2984. }
  2985. max_reclaim = min(reserved, to_reclaim);
  2986. nr_pages = max_t(unsigned long, nr_pages,
  2987. max_reclaim >> PAGE_CACHE_SHIFT);
  2988. while (loops < 1024) {
  2989. /* have the flusher threads jump in and do some IO */
  2990. smp_mb();
  2991. nr_pages = min_t(unsigned long, nr_pages,
  2992. root->fs_info->delalloc_bytes >> PAGE_CACHE_SHIFT);
  2993. writeback_inodes_sb_nr_if_idle(root->fs_info->sb, nr_pages);
  2994. spin_lock(&space_info->lock);
  2995. if (reserved > space_info->bytes_may_use)
  2996. reclaimed += reserved - space_info->bytes_may_use;
  2997. reserved = space_info->bytes_may_use;
  2998. spin_unlock(&space_info->lock);
  2999. loops++;
  3000. if (reserved == 0 || reclaimed >= max_reclaim)
  3001. break;
  3002. if (trans && trans->transaction->blocked)
  3003. return -EAGAIN;
  3004. if (wait_ordered && !trans) {
  3005. btrfs_wait_ordered_extents(root, 0, 0);
  3006. } else {
  3007. time_left = schedule_timeout_interruptible(1);
  3008. /* We were interrupted, exit */
  3009. if (time_left)
  3010. break;
  3011. }
  3012. /* we've kicked the IO a few times, if anything has been freed,
  3013. * exit. There is no sense in looping here for a long time
  3014. * when we really need to commit the transaction, or there are
  3015. * just too many writers without enough free space
  3016. */
  3017. if (loops > 3) {
  3018. smp_mb();
  3019. if (progress != space_info->reservation_progress)
  3020. break;
  3021. }
  3022. }
  3023. return reclaimed >= to_reclaim;
  3024. }
  3025. /**
  3026. * maybe_commit_transaction - possibly commit the transaction if its ok to
  3027. * @root - the root we're allocating for
  3028. * @bytes - the number of bytes we want to reserve
  3029. * @force - force the commit
  3030. *
  3031. * This will check to make sure that committing the transaction will actually
  3032. * get us somewhere and then commit the transaction if it does. Otherwise it
  3033. * will return -ENOSPC.
  3034. */
  3035. static int may_commit_transaction(struct btrfs_root *root,
  3036. struct btrfs_space_info *space_info,
  3037. u64 bytes, int force)
  3038. {
  3039. struct btrfs_block_rsv *delayed_rsv = &root->fs_info->delayed_block_rsv;
  3040. struct btrfs_trans_handle *trans;
  3041. trans = (struct btrfs_trans_handle *)current->journal_info;
  3042. if (trans)
  3043. return -EAGAIN;
  3044. if (force)
  3045. goto commit;
  3046. /* See if there is enough pinned space to make this reservation */
  3047. spin_lock(&space_info->lock);
  3048. if (space_info->bytes_pinned >= bytes) {
  3049. spin_unlock(&space_info->lock);
  3050. goto commit;
  3051. }
  3052. spin_unlock(&space_info->lock);
  3053. /*
  3054. * See if there is some space in the delayed insertion reservation for
  3055. * this reservation.
  3056. */
  3057. if (space_info != delayed_rsv->space_info)
  3058. return -ENOSPC;
  3059. spin_lock(&delayed_rsv->lock);
  3060. if (delayed_rsv->size < bytes) {
  3061. spin_unlock(&delayed_rsv->lock);
  3062. return -ENOSPC;
  3063. }
  3064. spin_unlock(&delayed_rsv->lock);
  3065. commit:
  3066. trans = btrfs_join_transaction(root);
  3067. if (IS_ERR(trans))
  3068. return -ENOSPC;
  3069. return btrfs_commit_transaction(trans, root);
  3070. }
  3071. /**
  3072. * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space
  3073. * @root - the root we're allocating for
  3074. * @block_rsv - the block_rsv we're allocating for
  3075. * @orig_bytes - the number of bytes we want
  3076. * @flush - wether or not we can flush to make our reservation
  3077. *
  3078. * This will reserve orgi_bytes number of bytes from the space info associated
  3079. * with the block_rsv. If there is not enough space it will make an attempt to
  3080. * flush out space to make room. It will do this by flushing delalloc if
  3081. * possible or committing the transaction. If flush is 0 then no attempts to
  3082. * regain reservations will be made and this will fail if there is not enough
  3083. * space already.
  3084. */
  3085. static int reserve_metadata_bytes(struct btrfs_root *root,
  3086. struct btrfs_block_rsv *block_rsv,
  3087. u64 orig_bytes, int flush)
  3088. {
  3089. struct btrfs_space_info *space_info = block_rsv->space_info;
  3090. u64 used;
  3091. u64 num_bytes = orig_bytes;
  3092. int retries = 0;
  3093. int ret = 0;
  3094. bool committed = false;
  3095. bool flushing = false;
  3096. bool wait_ordered = false;
  3097. again:
  3098. ret = 0;
  3099. spin_lock(&space_info->lock);
  3100. /*
  3101. * We only want to wait if somebody other than us is flushing and we are
  3102. * actually alloed to flush.
  3103. */
  3104. while (flush && !flushing && space_info->flush) {
  3105. spin_unlock(&space_info->lock);
  3106. /*
  3107. * If we have a trans handle we can't wait because the flusher
  3108. * may have to commit the transaction, which would mean we would
  3109. * deadlock since we are waiting for the flusher to finish, but
  3110. * hold the current transaction open.
  3111. */
  3112. if (current->journal_info)
  3113. return -EAGAIN;
  3114. ret = wait_event_interruptible(space_info->wait,
  3115. !space_info->flush);
  3116. /* Must have been interrupted, return */
  3117. if (ret)
  3118. return -EINTR;
  3119. spin_lock(&space_info->lock);
  3120. }
  3121. ret = -ENOSPC;
  3122. used = space_info->bytes_used + space_info->bytes_reserved +
  3123. space_info->bytes_pinned + space_info->bytes_readonly +
  3124. space_info->bytes_may_use;
  3125. /*
  3126. * The idea here is that we've not already over-reserved the block group
  3127. * then we can go ahead and save our reservation first and then start
  3128. * flushing if we need to. Otherwise if we've already overcommitted
  3129. * lets start flushing stuff first and then come back and try to make
  3130. * our reservation.
  3131. */
  3132. if (used <= space_info->total_bytes) {
  3133. if (used + orig_bytes <= space_info->total_bytes) {
  3134. space_info->bytes_may_use += orig_bytes;
  3135. ret = 0;
  3136. } else {
  3137. /*
  3138. * Ok set num_bytes to orig_bytes since we aren't
  3139. * overocmmitted, this way we only try and reclaim what
  3140. * we need.
  3141. */
  3142. num_bytes = orig_bytes;
  3143. }
  3144. } else {
  3145. /*
  3146. * Ok we're over committed, set num_bytes to the overcommitted
  3147. * amount plus the amount of bytes that we need for this
  3148. * reservation.
  3149. */
  3150. wait_ordered = true;
  3151. num_bytes = used - space_info->total_bytes +
  3152. (orig_bytes * (retries + 1));
  3153. }
  3154. if (ret) {
  3155. u64 profile = btrfs_get_alloc_profile(root, 0);
  3156. u64 avail;
  3157. /*
  3158. * If we have a lot of space that's pinned, don't bother doing
  3159. * the overcommit dance yet and just commit the transaction.
  3160. */
  3161. avail = (space_info->total_bytes - space_info->bytes_used) * 8;
  3162. do_div(avail, 10);
  3163. if (space_info->bytes_pinned >= avail && flush && !committed) {
  3164. space_info->flush = 1;
  3165. flushing = true;
  3166. spin_unlock(&space_info->lock);
  3167. ret = may_commit_transaction(root, space_info,
  3168. orig_bytes, 1);
  3169. if (ret)
  3170. goto out;
  3171. committed = true;
  3172. goto again;
  3173. }
  3174. spin_lock(&root->fs_info->free_chunk_lock);
  3175. avail = root->fs_info->free_chunk_space;
  3176. /*
  3177. * If we have dup, raid1 or raid10 then only half of the free
  3178. * space is actually useable.
  3179. */
  3180. if (profile & (BTRFS_BLOCK_GROUP_DUP |
  3181. BTRFS_BLOCK_GROUP_RAID1 |
  3182. BTRFS_BLOCK_GROUP_RAID10))
  3183. avail >>= 1;
  3184. /*
  3185. * If we aren't flushing don't let us overcommit too much, say
  3186. * 1/8th of the space. If we can flush, let it overcommit up to
  3187. * 1/2 of the space.
  3188. */
  3189. if (flush)
  3190. avail >>= 3;
  3191. else
  3192. avail >>= 1;
  3193. spin_unlock(&root->fs_info->free_chunk_lock);
  3194. if (used + num_bytes < space_info->total_bytes + avail) {
  3195. space_info->bytes_may_use += orig_bytes;
  3196. ret = 0;
  3197. } else {
  3198. wait_ordered = true;
  3199. }
  3200. }
  3201. /*
  3202. * Couldn't make our reservation, save our place so while we're trying
  3203. * to reclaim space we can actually use it instead of somebody else
  3204. * stealing it from us.
  3205. */
  3206. if (ret && flush) {
  3207. flushing = true;
  3208. space_info->flush = 1;
  3209. }
  3210. spin_unlock(&space_info->lock);
  3211. if (!ret || !flush)
  3212. goto out;
  3213. /*
  3214. * We do synchronous shrinking since we don't actually unreserve
  3215. * metadata until after the IO is completed.
  3216. */
  3217. ret = shrink_delalloc(root, num_bytes, wait_ordered);
  3218. if (ret < 0)
  3219. goto out;
  3220. ret = 0;
  3221. /*
  3222. * So if we were overcommitted it's possible that somebody else flushed
  3223. * out enough space and we simply didn't have enough space to reclaim,
  3224. * so go back around and try again.
  3225. */
  3226. if (retries < 2) {
  3227. wait_ordered = true;
  3228. retries++;
  3229. goto again;
  3230. }
  3231. ret = -ENOSPC;
  3232. if (committed)
  3233. goto out;
  3234. ret = may_commit_transaction(root, space_info, orig_bytes, 0);
  3235. if (!ret) {
  3236. committed = true;
  3237. goto again;
  3238. }
  3239. out:
  3240. if (flushing) {
  3241. spin_lock(&space_info->lock);
  3242. space_info->flush = 0;
  3243. wake_up_all(&space_info->wait);
  3244. spin_unlock(&space_info->lock);
  3245. }
  3246. return ret;
  3247. }
  3248. static struct btrfs_block_rsv *get_block_rsv(struct btrfs_trans_handle *trans,
  3249. struct btrfs_root *root)
  3250. {
  3251. struct btrfs_block_rsv *block_rsv = NULL;
  3252. if (root->ref_cows || root == root->fs_info->csum_root)
  3253. block_rsv = trans->block_rsv;
  3254. if (!block_rsv)
  3255. block_rsv = root->block_rsv;
  3256. if (!block_rsv)
  3257. block_rsv = &root->fs_info->empty_block_rsv;
  3258. return block_rsv;
  3259. }
  3260. static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv,
  3261. u64 num_bytes)
  3262. {
  3263. int ret = -ENOSPC;
  3264. spin_lock(&block_rsv->lock);
  3265. if (block_rsv->reserved >= num_bytes) {
  3266. block_rsv->reserved -= num_bytes;
  3267. if (block_rsv->reserved < block_rsv->size)
  3268. block_rsv->full = 0;
  3269. ret = 0;
  3270. }
  3271. spin_unlock(&block_rsv->lock);
  3272. return ret;
  3273. }
  3274. static void block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv,
  3275. u64 num_bytes, int update_size)
  3276. {
  3277. spin_lock(&block_rsv->lock);
  3278. block_rsv->reserved += num_bytes;
  3279. if (update_size)
  3280. block_rsv->size += num_bytes;
  3281. else if (block_rsv->reserved >= block_rsv->size)
  3282. block_rsv->full = 1;
  3283. spin_unlock(&block_rsv->lock);
  3284. }
  3285. static void block_rsv_release_bytes(struct btrfs_block_rsv *block_rsv,
  3286. struct btrfs_block_rsv *dest, u64 num_bytes)
  3287. {
  3288. struct btrfs_space_info *space_info = block_rsv->space_info;
  3289. spin_lock(&block_rsv->lock);
  3290. if (num_bytes == (u64)-1)
  3291. num_bytes = block_rsv->size;
  3292. block_rsv->size -= num_bytes;
  3293. if (block_rsv->reserved >= block_rsv->size) {
  3294. num_bytes = block_rsv->reserved - block_rsv->size;
  3295. block_rsv->reserved = block_rsv->size;
  3296. block_rsv->full = 1;
  3297. } else {
  3298. num_bytes = 0;
  3299. }
  3300. spin_unlock(&block_rsv->lock);
  3301. if (num_bytes > 0) {
  3302. if (dest) {
  3303. spin_lock(&dest->lock);
  3304. if (!dest->full) {
  3305. u64 bytes_to_add;
  3306. bytes_to_add = dest->size - dest->reserved;
  3307. bytes_to_add = min(num_bytes, bytes_to_add);
  3308. dest->reserved += bytes_to_add;
  3309. if (dest->reserved >= dest->size)
  3310. dest->full = 1;
  3311. num_bytes -= bytes_to_add;
  3312. }
  3313. spin_unlock(&dest->lock);
  3314. }
  3315. if (num_bytes) {
  3316. spin_lock(&space_info->lock);
  3317. space_info->bytes_may_use -= num_bytes;
  3318. space_info->reservation_progress++;
  3319. spin_unlock(&space_info->lock);
  3320. }
  3321. }
  3322. }
  3323. static int block_rsv_migrate_bytes(struct btrfs_block_rsv *src,
  3324. struct btrfs_block_rsv *dst, u64 num_bytes)
  3325. {
  3326. int ret;
  3327. ret = block_rsv_use_bytes(src, num_bytes);
  3328. if (ret)
  3329. return ret;
  3330. block_rsv_add_bytes(dst, num_bytes, 1);
  3331. return 0;
  3332. }
  3333. void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv)
  3334. {
  3335. memset(rsv, 0, sizeof(*rsv));
  3336. spin_lock_init(&rsv->lock);
  3337. }
  3338. struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_root *root)
  3339. {
  3340. struct btrfs_block_rsv *block_rsv;
  3341. struct btrfs_fs_info *fs_info = root->fs_info;
  3342. block_rsv = kmalloc(sizeof(*block_rsv), GFP_NOFS);
  3343. if (!block_rsv)
  3344. return NULL;
  3345. btrfs_init_block_rsv(block_rsv);
  3346. block_rsv->space_info = __find_space_info(fs_info,
  3347. BTRFS_BLOCK_GROUP_METADATA);
  3348. return block_rsv;
  3349. }
  3350. void btrfs_free_block_rsv(struct btrfs_root *root,
  3351. struct btrfs_block_rsv *rsv)
  3352. {
  3353. btrfs_block_rsv_release(root, rsv, (u64)-1);
  3354. kfree(rsv);
  3355. }
  3356. int btrfs_block_rsv_add(struct btrfs_root *root,
  3357. struct btrfs_block_rsv *block_rsv,
  3358. u64 num_bytes)
  3359. {
  3360. int ret;
  3361. if (num_bytes == 0)
  3362. return 0;
  3363. ret = reserve_metadata_bytes(root, block_rsv, num_bytes, 1);
  3364. if (!ret) {
  3365. block_rsv_add_bytes(block_rsv, num_bytes, 1);
  3366. return 0;
  3367. }
  3368. return ret;
  3369. }
  3370. int btrfs_block_rsv_check(struct btrfs_root *root,
  3371. struct btrfs_block_rsv *block_rsv, int min_factor)
  3372. {
  3373. u64 num_bytes = 0;
  3374. int ret = -ENOSPC;
  3375. if (!block_rsv)
  3376. return 0;
  3377. spin_lock(&block_rsv->lock);
  3378. num_bytes = div_factor(block_rsv->size, min_factor);
  3379. if (block_rsv->reserved >= num_bytes)
  3380. ret = 0;
  3381. spin_unlock(&block_rsv->lock);
  3382. return ret;
  3383. }
  3384. int btrfs_block_rsv_refill(struct btrfs_root *root,
  3385. struct btrfs_block_rsv *block_rsv,
  3386. u64 min_reserved)
  3387. {
  3388. u64 num_bytes = 0;
  3389. int ret = -ENOSPC;
  3390. if (!block_rsv)
  3391. return 0;
  3392. spin_lock(&block_rsv->lock);
  3393. num_bytes = min_reserved;
  3394. if (block_rsv->reserved >= num_bytes)
  3395. ret = 0;
  3396. else
  3397. num_bytes -= block_rsv->reserved;
  3398. spin_unlock(&block_rsv->lock);
  3399. if (!ret)
  3400. return 0;
  3401. ret = reserve_metadata_bytes(root, block_rsv, num_bytes, 1);
  3402. if (!ret) {
  3403. block_rsv_add_bytes(block_rsv, num_bytes, 0);
  3404. return 0;
  3405. }
  3406. return ret;
  3407. }
  3408. int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src_rsv,
  3409. struct btrfs_block_rsv *dst_rsv,
  3410. u64 num_bytes)
  3411. {
  3412. return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
  3413. }
  3414. void btrfs_block_rsv_release(struct btrfs_root *root,
  3415. struct btrfs_block_rsv *block_rsv,
  3416. u64 num_bytes)
  3417. {
  3418. struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
  3419. if (global_rsv->full || global_rsv == block_rsv ||
  3420. block_rsv->space_info != global_rsv->space_info)
  3421. global_rsv = NULL;
  3422. block_rsv_release_bytes(block_rsv, global_rsv, num_bytes);
  3423. }
  3424. /*
  3425. * helper to calculate size of global block reservation.
  3426. * the desired value is sum of space used by extent tree,
  3427. * checksum tree and root tree
  3428. */
  3429. static u64 calc_global_metadata_size(struct btrfs_fs_info *fs_info)
  3430. {
  3431. struct btrfs_space_info *sinfo;
  3432. u64 num_bytes;
  3433. u64 meta_used;
  3434. u64 data_used;
  3435. int csum_size = btrfs_super_csum_size(fs_info->super_copy);
  3436. sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_DATA);
  3437. spin_lock(&sinfo->lock);
  3438. data_used = sinfo->bytes_used;
  3439. spin_unlock(&sinfo->lock);
  3440. sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
  3441. spin_lock(&sinfo->lock);
  3442. if (sinfo->flags & BTRFS_BLOCK_GROUP_DATA)
  3443. data_used = 0;
  3444. meta_used = sinfo->bytes_used;
  3445. spin_unlock(&sinfo->lock);
  3446. num_bytes = (data_used >> fs_info->sb->s_blocksize_bits) *
  3447. csum_size * 2;
  3448. num_bytes += div64_u64(data_used + meta_used, 50);
  3449. if (num_bytes * 3 > meta_used)
  3450. num_bytes = div64_u64(meta_used, 3);
  3451. return ALIGN(num_bytes, fs_info->extent_root->leafsize << 10);
  3452. }
  3453. static void update_global_block_rsv(struct btrfs_fs_info *fs_info)
  3454. {
  3455. struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv;
  3456. struct btrfs_space_info *sinfo = block_rsv->space_info;
  3457. u64 num_bytes;
  3458. num_bytes = calc_global_metadata_size(fs_info);
  3459. spin_lock(&block_rsv->lock);
  3460. spin_lock(&sinfo->lock);
  3461. block_rsv->size = num_bytes;
  3462. num_bytes = sinfo->bytes_used + sinfo->bytes_pinned +
  3463. sinfo->bytes_reserved + sinfo->bytes_readonly +
  3464. sinfo->bytes_may_use;
  3465. if (sinfo->total_bytes > num_bytes) {
  3466. num_bytes = sinfo->total_bytes - num_bytes;
  3467. block_rsv->reserved += num_bytes;
  3468. sinfo->bytes_may_use += num_bytes;
  3469. }
  3470. if (block_rsv->reserved >= block_rsv->size) {
  3471. num_bytes = block_rsv->reserved - block_rsv->size;
  3472. sinfo->bytes_may_use -= num_bytes;
  3473. sinfo->reservation_progress++;
  3474. block_rsv->reserved = block_rsv->size;
  3475. block_rsv->full = 1;
  3476. }
  3477. spin_unlock(&sinfo->lock);
  3478. spin_unlock(&block_rsv->lock);
  3479. }
  3480. static void init_global_block_rsv(struct btrfs_fs_info *fs_info)
  3481. {
  3482. struct btrfs_space_info *space_info;
  3483. space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
  3484. fs_info->chunk_block_rsv.space_info = space_info;
  3485. space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
  3486. fs_info->global_block_rsv.space_info = space_info;
  3487. fs_info->delalloc_block_rsv.space_info = space_info;
  3488. fs_info->trans_block_rsv.space_info = space_info;
  3489. fs_info->empty_block_rsv.space_info = space_info;
  3490. fs_info->delayed_block_rsv.space_info = space_info;
  3491. fs_info->extent_root->block_rsv = &fs_info->global_block_rsv;
  3492. fs_info->csum_root->block_rsv = &fs_info->global_block_rsv;
  3493. fs_info->dev_root->block_rsv = &fs_info->global_block_rsv;
  3494. fs_info->tree_root->block_rsv = &fs_info->global_block_rsv;
  3495. fs_info->chunk_root->block_rsv = &fs_info->chunk_block_rsv;
  3496. update_global_block_rsv(fs_info);
  3497. }
  3498. static void release_global_block_rsv(struct btrfs_fs_info *fs_info)
  3499. {
  3500. block_rsv_release_bytes(&fs_info->global_block_rsv, NULL, (u64)-1);
  3501. WARN_ON(fs_info->delalloc_block_rsv.size > 0);
  3502. WARN_ON(fs_info->delalloc_block_rsv.reserved > 0);
  3503. WARN_ON(fs_info->trans_block_rsv.size > 0);
  3504. WARN_ON(fs_info->trans_block_rsv.reserved > 0);
  3505. WARN_ON(fs_info->chunk_block_rsv.size > 0);
  3506. WARN_ON(fs_info->chunk_block_rsv.reserved > 0);
  3507. WARN_ON(fs_info->delayed_block_rsv.size > 0);
  3508. WARN_ON(fs_info->delayed_block_rsv.reserved > 0);
  3509. }
  3510. void btrfs_trans_release_metadata(struct btrfs_trans_handle *trans,
  3511. struct btrfs_root *root)
  3512. {
  3513. if (!trans->bytes_reserved)
  3514. return;
  3515. btrfs_block_rsv_release(root, trans->block_rsv, trans->bytes_reserved);
  3516. trans->bytes_reserved = 0;
  3517. }
  3518. int btrfs_orphan_reserve_metadata(struct btrfs_trans_handle *trans,
  3519. struct inode *inode)
  3520. {
  3521. struct btrfs_root *root = BTRFS_I(inode)->root;
  3522. struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
  3523. struct btrfs_block_rsv *dst_rsv = root->orphan_block_rsv;
  3524. /*
  3525. * We need to hold space in order to delete our orphan item once we've
  3526. * added it, so this takes the reservation so we can release it later
  3527. * when we are truly done with the orphan item.
  3528. */
  3529. u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
  3530. return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
  3531. }
  3532. void btrfs_orphan_release_metadata(struct inode *inode)
  3533. {
  3534. struct btrfs_root *root = BTRFS_I(inode)->root;
  3535. u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
  3536. btrfs_block_rsv_release(root, root->orphan_block_rsv, num_bytes);
  3537. }
  3538. int btrfs_snap_reserve_metadata(struct btrfs_trans_handle *trans,
  3539. struct btrfs_pending_snapshot *pending)
  3540. {
  3541. struct btrfs_root *root = pending->root;
  3542. struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
  3543. struct btrfs_block_rsv *dst_rsv = &pending->block_rsv;
  3544. /*
  3545. * two for root back/forward refs, two for directory entries
  3546. * and one for root of the snapshot.
  3547. */
  3548. u64 num_bytes = btrfs_calc_trans_metadata_size(root, 5);
  3549. dst_rsv->space_info = src_rsv->space_info;
  3550. return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
  3551. }
  3552. /**
  3553. * drop_outstanding_extent - drop an outstanding extent
  3554. * @inode: the inode we're dropping the extent for
  3555. *
  3556. * This is called when we are freeing up an outstanding extent, either called
  3557. * after an error or after an extent is written. This will return the number of
  3558. * reserved extents that need to be freed. This must be called with
  3559. * BTRFS_I(inode)->lock held.
  3560. */
  3561. static unsigned drop_outstanding_extent(struct inode *inode)
  3562. {
  3563. unsigned dropped_extents = 0;
  3564. BUG_ON(!BTRFS_I(inode)->outstanding_extents);
  3565. BTRFS_I(inode)->outstanding_extents--;
  3566. /*
  3567. * If we have more or the same amount of outsanding extents than we have
  3568. * reserved then we need to leave the reserved extents count alone.
  3569. */
  3570. if (BTRFS_I(inode)->outstanding_extents >=
  3571. BTRFS_I(inode)->reserved_extents)
  3572. return 0;
  3573. dropped_extents = BTRFS_I(inode)->reserved_extents -
  3574. BTRFS_I(inode)->outstanding_extents;
  3575. BTRFS_I(inode)->reserved_extents -= dropped_extents;
  3576. return dropped_extents;
  3577. }
  3578. /**
  3579. * calc_csum_metadata_size - return the amount of metada space that must be
  3580. * reserved/free'd for the given bytes.
  3581. * @inode: the inode we're manipulating
  3582. * @num_bytes: the number of bytes in question
  3583. * @reserve: 1 if we are reserving space, 0 if we are freeing space
  3584. *
  3585. * This adjusts the number of csum_bytes in the inode and then returns the
  3586. * correct amount of metadata that must either be reserved or freed. We
  3587. * calculate how many checksums we can fit into one leaf and then divide the
  3588. * number of bytes that will need to be checksumed by this value to figure out
  3589. * how many checksums will be required. If we are adding bytes then the number
  3590. * may go up and we will return the number of additional bytes that must be
  3591. * reserved. If it is going down we will return the number of bytes that must
  3592. * be freed.
  3593. *
  3594. * This must be called with BTRFS_I(inode)->lock held.
  3595. */
  3596. static u64 calc_csum_metadata_size(struct inode *inode, u64 num_bytes,
  3597. int reserve)
  3598. {
  3599. struct btrfs_root *root = BTRFS_I(inode)->root;
  3600. u64 csum_size;
  3601. int num_csums_per_leaf;
  3602. int num_csums;
  3603. int old_csums;
  3604. if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM &&
  3605. BTRFS_I(inode)->csum_bytes == 0)
  3606. return 0;
  3607. old_csums = (int)div64_u64(BTRFS_I(inode)->csum_bytes, root->sectorsize);
  3608. if (reserve)
  3609. BTRFS_I(inode)->csum_bytes += num_bytes;
  3610. else
  3611. BTRFS_I(inode)->csum_bytes -= num_bytes;
  3612. csum_size = BTRFS_LEAF_DATA_SIZE(root) - sizeof(struct btrfs_item);
  3613. num_csums_per_leaf = (int)div64_u64(csum_size,
  3614. sizeof(struct btrfs_csum_item) +
  3615. sizeof(struct btrfs_disk_key));
  3616. num_csums = (int)div64_u64(BTRFS_I(inode)->csum_bytes, root->sectorsize);
  3617. num_csums = num_csums + num_csums_per_leaf - 1;
  3618. num_csums = num_csums / num_csums_per_leaf;
  3619. old_csums = old_csums + num_csums_per_leaf - 1;
  3620. old_csums = old_csums / num_csums_per_leaf;
  3621. /* No change, no need to reserve more */
  3622. if (old_csums == num_csums)
  3623. return 0;
  3624. if (reserve)
  3625. return btrfs_calc_trans_metadata_size(root,
  3626. num_csums - old_csums);
  3627. return btrfs_calc_trans_metadata_size(root, old_csums - num_csums);
  3628. }
  3629. int btrfs_delalloc_reserve_metadata(struct inode *inode, u64 num_bytes)
  3630. {
  3631. struct btrfs_root *root = BTRFS_I(inode)->root;
  3632. struct btrfs_block_rsv *block_rsv = &root->fs_info->delalloc_block_rsv;
  3633. u64 to_reserve = 0;
  3634. unsigned nr_extents = 0;
  3635. int flush = 1;
  3636. int ret;
  3637. if (btrfs_is_free_space_inode(root, inode))
  3638. flush = 0;
  3639. if (flush && btrfs_transaction_in_commit(root->fs_info))
  3640. schedule_timeout(1);
  3641. num_bytes = ALIGN(num_bytes, root->sectorsize);
  3642. spin_lock(&BTRFS_I(inode)->lock);
  3643. BTRFS_I(inode)->outstanding_extents++;
  3644. if (BTRFS_I(inode)->outstanding_extents >
  3645. BTRFS_I(inode)->reserved_extents) {
  3646. nr_extents = BTRFS_I(inode)->outstanding_extents -
  3647. BTRFS_I(inode)->reserved_extents;
  3648. BTRFS_I(inode)->reserved_extents += nr_extents;
  3649. to_reserve = btrfs_calc_trans_metadata_size(root, nr_extents);
  3650. }
  3651. to_reserve += calc_csum_metadata_size(inode, num_bytes, 1);
  3652. spin_unlock(&BTRFS_I(inode)->lock);
  3653. ret = reserve_metadata_bytes(root, block_rsv, to_reserve, flush);
  3654. if (ret) {
  3655. u64 to_free = 0;
  3656. unsigned dropped;
  3657. spin_lock(&BTRFS_I(inode)->lock);
  3658. dropped = drop_outstanding_extent(inode);
  3659. to_free = calc_csum_metadata_size(inode, num_bytes, 0);
  3660. spin_unlock(&BTRFS_I(inode)->lock);
  3661. to_free += btrfs_calc_trans_metadata_size(root, dropped);
  3662. /*
  3663. * Somebody could have come in and twiddled with the
  3664. * reservation, so if we have to free more than we would have
  3665. * reserved from this reservation go ahead and release those
  3666. * bytes.
  3667. */
  3668. to_free -= to_reserve;
  3669. if (to_free)
  3670. btrfs_block_rsv_release(root, block_rsv, to_free);
  3671. return ret;
  3672. }
  3673. block_rsv_add_bytes(block_rsv, to_reserve, 1);
  3674. return 0;
  3675. }
  3676. /**
  3677. * btrfs_delalloc_release_metadata - release a metadata reservation for an inode
  3678. * @inode: the inode to release the reservation for
  3679. * @num_bytes: the number of bytes we're releasing
  3680. *
  3681. * This will release the metadata reservation for an inode. This can be called
  3682. * once we complete IO for a given set of bytes to release their metadata
  3683. * reservations.
  3684. */
  3685. void btrfs_delalloc_release_metadata(struct inode *inode, u64 num_bytes)
  3686. {
  3687. struct btrfs_root *root = BTRFS_I(inode)->root;
  3688. u64 to_free = 0;
  3689. unsigned dropped;
  3690. num_bytes = ALIGN(num_bytes, root->sectorsize);
  3691. spin_lock(&BTRFS_I(inode)->lock);
  3692. dropped = drop_outstanding_extent(inode);
  3693. to_free = calc_csum_metadata_size(inode, num_bytes, 0);
  3694. spin_unlock(&BTRFS_I(inode)->lock);
  3695. if (dropped > 0)
  3696. to_free += btrfs_calc_trans_metadata_size(root, dropped);
  3697. btrfs_block_rsv_release(root, &root->fs_info->delalloc_block_rsv,
  3698. to_free);
  3699. }
  3700. /**
  3701. * btrfs_delalloc_reserve_space - reserve data and metadata space for delalloc
  3702. * @inode: inode we're writing to
  3703. * @num_bytes: the number of bytes we want to allocate
  3704. *
  3705. * This will do the following things
  3706. *
  3707. * o reserve space in the data space info for num_bytes
  3708. * o reserve space in the metadata space info based on number of outstanding
  3709. * extents and how much csums will be needed
  3710. * o add to the inodes ->delalloc_bytes
  3711. * o add it to the fs_info's delalloc inodes list.
  3712. *
  3713. * This will return 0 for success and -ENOSPC if there is no space left.
  3714. */
  3715. int btrfs_delalloc_reserve_space(struct inode *inode, u64 num_bytes)
  3716. {
  3717. int ret;
  3718. ret = btrfs_check_data_free_space(inode, num_bytes);
  3719. if (ret)
  3720. return ret;
  3721. ret = btrfs_delalloc_reserve_metadata(inode, num_bytes);
  3722. if (ret) {
  3723. btrfs_free_reserved_data_space(inode, num_bytes);
  3724. return ret;
  3725. }
  3726. return 0;
  3727. }
  3728. /**
  3729. * btrfs_delalloc_release_space - release data and metadata space for delalloc
  3730. * @inode: inode we're releasing space for
  3731. * @num_bytes: the number of bytes we want to free up
  3732. *
  3733. * This must be matched with a call to btrfs_delalloc_reserve_space. This is
  3734. * called in the case that we don't need the metadata AND data reservations
  3735. * anymore. So if there is an error or we insert an inline extent.
  3736. *
  3737. * This function will release the metadata space that was not used and will
  3738. * decrement ->delalloc_bytes and remove it from the fs_info delalloc_inodes
  3739. * list if there are no delalloc bytes left.
  3740. */
  3741. void btrfs_delalloc_release_space(struct inode *inode, u64 num_bytes)
  3742. {
  3743. btrfs_delalloc_release_metadata(inode, num_bytes);
  3744. btrfs_free_reserved_data_space(inode, num_bytes);
  3745. }
  3746. static int update_block_group(struct btrfs_trans_handle *trans,
  3747. struct btrfs_root *root,
  3748. u64 bytenr, u64 num_bytes, int alloc)
  3749. {
  3750. struct btrfs_block_group_cache *cache = NULL;
  3751. struct btrfs_fs_info *info = root->fs_info;
  3752. u64 total = num_bytes;
  3753. u64 old_val;
  3754. u64 byte_in_group;
  3755. int factor;
  3756. /* block accounting for super block */
  3757. spin_lock(&info->delalloc_lock);
  3758. old_val = btrfs_super_bytes_used(info->super_copy);
  3759. if (alloc)
  3760. old_val += num_bytes;
  3761. else
  3762. old_val -= num_bytes;
  3763. btrfs_set_super_bytes_used(info->super_copy, old_val);
  3764. spin_unlock(&info->delalloc_lock);
  3765. while (total) {
  3766. cache = btrfs_lookup_block_group(info, bytenr);
  3767. if (!cache)
  3768. return -1;
  3769. if (cache->flags & (BTRFS_BLOCK_GROUP_DUP |
  3770. BTRFS_BLOCK_GROUP_RAID1 |
  3771. BTRFS_BLOCK_GROUP_RAID10))
  3772. factor = 2;
  3773. else
  3774. factor = 1;
  3775. /*
  3776. * If this block group has free space cache written out, we
  3777. * need to make sure to load it if we are removing space. This
  3778. * is because we need the unpinning stage to actually add the
  3779. * space back to the block group, otherwise we will leak space.
  3780. */
  3781. if (!alloc && cache->cached == BTRFS_CACHE_NO)
  3782. cache_block_group(cache, trans, NULL, 1);
  3783. byte_in_group = bytenr - cache->key.objectid;
  3784. WARN_ON(byte_in_group > cache->key.offset);
  3785. spin_lock(&cache->space_info->lock);
  3786. spin_lock(&cache->lock);
  3787. if (btrfs_test_opt(root, SPACE_CACHE) &&
  3788. cache->disk_cache_state < BTRFS_DC_CLEAR)
  3789. cache->disk_cache_state = BTRFS_DC_CLEAR;
  3790. cache->dirty = 1;
  3791. old_val = btrfs_block_group_used(&cache->item);
  3792. num_bytes = min(total, cache->key.offset - byte_in_group);
  3793. if (alloc) {
  3794. old_val += num_bytes;
  3795. btrfs_set_block_group_used(&cache->item, old_val);
  3796. cache->reserved -= num_bytes;
  3797. cache->space_info->bytes_reserved -= num_bytes;
  3798. cache->space_info->bytes_used += num_bytes;
  3799. cache->space_info->disk_used += num_bytes * factor;
  3800. spin_unlock(&cache->lock);
  3801. spin_unlock(&cache->space_info->lock);
  3802. } else {
  3803. old_val -= num_bytes;
  3804. btrfs_set_block_group_used(&cache->item, old_val);
  3805. cache->pinned += num_bytes;
  3806. cache->space_info->bytes_pinned += num_bytes;
  3807. cache->space_info->bytes_used -= num_bytes;
  3808. cache->space_info->disk_used -= num_bytes * factor;
  3809. spin_unlock(&cache->lock);
  3810. spin_unlock(&cache->space_info->lock);
  3811. set_extent_dirty(info->pinned_extents,
  3812. bytenr, bytenr + num_bytes - 1,
  3813. GFP_NOFS | __GFP_NOFAIL);
  3814. }
  3815. btrfs_put_block_group(cache);
  3816. total -= num_bytes;
  3817. bytenr += num_bytes;
  3818. }
  3819. return 0;
  3820. }
  3821. static u64 first_logical_byte(struct btrfs_root *root, u64 search_start)
  3822. {
  3823. struct btrfs_block_group_cache *cache;
  3824. u64 bytenr;
  3825. cache = btrfs_lookup_first_block_group(root->fs_info, search_start);
  3826. if (!cache)
  3827. return 0;
  3828. bytenr = cache->key.objectid;
  3829. btrfs_put_block_group(cache);
  3830. return bytenr;
  3831. }
  3832. static int pin_down_extent(struct btrfs_root *root,
  3833. struct btrfs_block_group_cache *cache,
  3834. u64 bytenr, u64 num_bytes, int reserved)
  3835. {
  3836. spin_lock(&cache->space_info->lock);
  3837. spin_lock(&cache->lock);
  3838. cache->pinned += num_bytes;
  3839. cache->space_info->bytes_pinned += num_bytes;
  3840. if (reserved) {
  3841. cache->reserved -= num_bytes;
  3842. cache->space_info->bytes_reserved -= num_bytes;
  3843. }
  3844. spin_unlock(&cache->lock);
  3845. spin_unlock(&cache->space_info->lock);
  3846. set_extent_dirty(root->fs_info->pinned_extents, bytenr,
  3847. bytenr + num_bytes - 1, GFP_NOFS | __GFP_NOFAIL);
  3848. return 0;
  3849. }
  3850. /*
  3851. * this function must be called within transaction
  3852. */
  3853. int btrfs_pin_extent(struct btrfs_root *root,
  3854. u64 bytenr, u64 num_bytes, int reserved)
  3855. {
  3856. struct btrfs_block_group_cache *cache;
  3857. cache = btrfs_lookup_block_group(root->fs_info, bytenr);
  3858. BUG_ON(!cache);
  3859. pin_down_extent(root, cache, bytenr, num_bytes, reserved);
  3860. btrfs_put_block_group(cache);
  3861. return 0;
  3862. }
  3863. /*
  3864. * this function must be called within transaction
  3865. */
  3866. int btrfs_pin_extent_for_log_replay(struct btrfs_trans_handle *trans,
  3867. struct btrfs_root *root,
  3868. u64 bytenr, u64 num_bytes)
  3869. {
  3870. struct btrfs_block_group_cache *cache;
  3871. cache = btrfs_lookup_block_group(root->fs_info, bytenr);
  3872. BUG_ON(!cache);
  3873. /*
  3874. * pull in the free space cache (if any) so that our pin
  3875. * removes the free space from the cache. We have load_only set
  3876. * to one because the slow code to read in the free extents does check
  3877. * the pinned extents.
  3878. */
  3879. cache_block_group(cache, trans, root, 1);
  3880. pin_down_extent(root, cache, bytenr, num_bytes, 0);
  3881. /* remove us from the free space cache (if we're there at all) */
  3882. btrfs_remove_free_space(cache, bytenr, num_bytes);
  3883. btrfs_put_block_group(cache);
  3884. return 0;
  3885. }
  3886. /**
  3887. * btrfs_update_reserved_bytes - update the block_group and space info counters
  3888. * @cache: The cache we are manipulating
  3889. * @num_bytes: The number of bytes in question
  3890. * @reserve: One of the reservation enums
  3891. *
  3892. * This is called by the allocator when it reserves space, or by somebody who is
  3893. * freeing space that was never actually used on disk. For example if you
  3894. * reserve some space for a new leaf in transaction A and before transaction A
  3895. * commits you free that leaf, you call this with reserve set to 0 in order to
  3896. * clear the reservation.
  3897. *
  3898. * Metadata reservations should be called with RESERVE_ALLOC so we do the proper
  3899. * ENOSPC accounting. For data we handle the reservation through clearing the
  3900. * delalloc bits in the io_tree. We have to do this since we could end up
  3901. * allocating less disk space for the amount of data we have reserved in the
  3902. * case of compression.
  3903. *
  3904. * If this is a reservation and the block group has become read only we cannot
  3905. * make the reservation and return -EAGAIN, otherwise this function always
  3906. * succeeds.
  3907. */
  3908. static int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache,
  3909. u64 num_bytes, int reserve)
  3910. {
  3911. struct btrfs_space_info *space_info = cache->space_info;
  3912. int ret = 0;
  3913. spin_lock(&space_info->lock);
  3914. spin_lock(&cache->lock);
  3915. if (reserve != RESERVE_FREE) {
  3916. if (cache->ro) {
  3917. ret = -EAGAIN;
  3918. } else {
  3919. cache->reserved += num_bytes;
  3920. space_info->bytes_reserved += num_bytes;
  3921. if (reserve == RESERVE_ALLOC) {
  3922. BUG_ON(space_info->bytes_may_use < num_bytes);
  3923. space_info->bytes_may_use -= num_bytes;
  3924. }
  3925. }
  3926. } else {
  3927. if (cache->ro)
  3928. space_info->bytes_readonly += num_bytes;
  3929. cache->reserved -= num_bytes;
  3930. space_info->bytes_reserved -= num_bytes;
  3931. space_info->reservation_progress++;
  3932. }
  3933. spin_unlock(&cache->lock);
  3934. spin_unlock(&space_info->lock);
  3935. return ret;
  3936. }
  3937. int btrfs_prepare_extent_commit(struct btrfs_trans_handle *trans,
  3938. struct btrfs_root *root)
  3939. {
  3940. struct btrfs_fs_info *fs_info = root->fs_info;
  3941. struct btrfs_caching_control *next;
  3942. struct btrfs_caching_control *caching_ctl;
  3943. struct btrfs_block_group_cache *cache;
  3944. down_write(&fs_info->extent_commit_sem);
  3945. list_for_each_entry_safe(caching_ctl, next,
  3946. &fs_info->caching_block_groups, list) {
  3947. cache = caching_ctl->block_group;
  3948. if (block_group_cache_done(cache)) {
  3949. cache->last_byte_to_unpin = (u64)-1;
  3950. list_del_init(&caching_ctl->list);
  3951. put_caching_control(caching_ctl);
  3952. } else {
  3953. cache->last_byte_to_unpin = caching_ctl->progress;
  3954. }
  3955. }
  3956. if (fs_info->pinned_extents == &fs_info->freed_extents[0])
  3957. fs_info->pinned_extents = &fs_info->freed_extents[1];
  3958. else
  3959. fs_info->pinned_extents = &fs_info->freed_extents[0];
  3960. up_write(&fs_info->extent_commit_sem);
  3961. update_global_block_rsv(fs_info);
  3962. return 0;
  3963. }
  3964. static int unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
  3965. {
  3966. struct btrfs_fs_info *fs_info = root->fs_info;
  3967. struct btrfs_block_group_cache *cache = NULL;
  3968. u64 len;
  3969. while (start <= end) {
  3970. if (!cache ||
  3971. start >= cache->key.objectid + cache->key.offset) {
  3972. if (cache)
  3973. btrfs_put_block_group(cache);
  3974. cache = btrfs_lookup_block_group(fs_info, start);
  3975. BUG_ON(!cache);
  3976. }
  3977. len = cache->key.objectid + cache->key.offset - start;
  3978. len = min(len, end + 1 - start);
  3979. if (start < cache->last_byte_to_unpin) {
  3980. len = min(len, cache->last_byte_to_unpin - start);
  3981. btrfs_add_free_space(cache, start, len);
  3982. }
  3983. start += len;
  3984. spin_lock(&cache->space_info->lock);
  3985. spin_lock(&cache->lock);
  3986. cache->pinned -= len;
  3987. cache->space_info->bytes_pinned -= len;
  3988. if (cache->ro)
  3989. cache->space_info->bytes_readonly += len;
  3990. spin_unlock(&cache->lock);
  3991. spin_unlock(&cache->space_info->lock);
  3992. }
  3993. if (cache)
  3994. btrfs_put_block_group(cache);
  3995. return 0;
  3996. }
  3997. int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans,
  3998. struct btrfs_root *root)
  3999. {
  4000. struct btrfs_fs_info *fs_info = root->fs_info;
  4001. struct extent_io_tree *unpin;
  4002. u64 start;
  4003. u64 end;
  4004. int ret;
  4005. if (fs_info->pinned_extents == &fs_info->freed_extents[0])
  4006. unpin = &fs_info->freed_extents[1];
  4007. else
  4008. unpin = &fs_info->freed_extents[0];
  4009. while (1) {
  4010. ret = find_first_extent_bit(unpin, 0, &start, &end,
  4011. EXTENT_DIRTY);
  4012. if (ret)
  4013. break;
  4014. if (btrfs_test_opt(root, DISCARD))
  4015. ret = btrfs_discard_extent(root, start,
  4016. end + 1 - start, NULL);
  4017. clear_extent_dirty(unpin, start, end, GFP_NOFS);
  4018. unpin_extent_range(root, start, end);
  4019. cond_resched();
  4020. }
  4021. return 0;
  4022. }
  4023. static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
  4024. struct btrfs_root *root,
  4025. u64 bytenr, u64 num_bytes, u64 parent,
  4026. u64 root_objectid, u64 owner_objectid,
  4027. u64 owner_offset, int refs_to_drop,
  4028. struct btrfs_delayed_extent_op *extent_op)
  4029. {
  4030. struct btrfs_key key;
  4031. struct btrfs_path *path;
  4032. struct btrfs_fs_info *info = root->fs_info;
  4033. struct btrfs_root *extent_root = info->extent_root;
  4034. struct extent_buffer *leaf;
  4035. struct btrfs_extent_item *ei;
  4036. struct btrfs_extent_inline_ref *iref;
  4037. int ret;
  4038. int is_data;
  4039. int extent_slot = 0;
  4040. int found_extent = 0;
  4041. int num_to_del = 1;
  4042. u32 item_size;
  4043. u64 refs;
  4044. path = btrfs_alloc_path();
  4045. if (!path)
  4046. return -ENOMEM;
  4047. path->reada = 1;
  4048. path->leave_spinning = 1;
  4049. is_data = owner_objectid >= BTRFS_FIRST_FREE_OBJECTID;
  4050. BUG_ON(!is_data && refs_to_drop != 1);
  4051. ret = lookup_extent_backref(trans, extent_root, path, &iref,
  4052. bytenr, num_bytes, parent,
  4053. root_objectid, owner_objectid,
  4054. owner_offset);
  4055. if (ret == 0) {
  4056. extent_slot = path->slots[0];
  4057. while (extent_slot >= 0) {
  4058. btrfs_item_key_to_cpu(path->nodes[0], &key,
  4059. extent_slot);
  4060. if (key.objectid != bytenr)
  4061. break;
  4062. if (key.type == BTRFS_EXTENT_ITEM_KEY &&
  4063. key.offset == num_bytes) {
  4064. found_extent = 1;
  4065. break;
  4066. }
  4067. if (path->slots[0] - extent_slot > 5)
  4068. break;
  4069. extent_slot--;
  4070. }
  4071. #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
  4072. item_size = btrfs_item_size_nr(path->nodes[0], extent_slot);
  4073. if (found_extent && item_size < sizeof(*ei))
  4074. found_extent = 0;
  4075. #endif
  4076. if (!found_extent) {
  4077. BUG_ON(iref);
  4078. ret = remove_extent_backref(trans, extent_root, path,
  4079. NULL, refs_to_drop,
  4080. is_data);
  4081. BUG_ON(ret);
  4082. btrfs_release_path(path);
  4083. path->leave_spinning = 1;
  4084. key.objectid = bytenr;
  4085. key.type = BTRFS_EXTENT_ITEM_KEY;
  4086. key.offset = num_bytes;
  4087. ret = btrfs_search_slot(trans, extent_root,
  4088. &key, path, -1, 1);
  4089. if (ret) {
  4090. printk(KERN_ERR "umm, got %d back from search"
  4091. ", was looking for %llu\n", ret,
  4092. (unsigned long long)bytenr);
  4093. if (ret > 0)
  4094. btrfs_print_leaf(extent_root,
  4095. path->nodes[0]);
  4096. }
  4097. BUG_ON(ret);
  4098. extent_slot = path->slots[0];
  4099. }
  4100. } else {
  4101. btrfs_print_leaf(extent_root, path->nodes[0]);
  4102. WARN_ON(1);
  4103. printk(KERN_ERR "btrfs unable to find ref byte nr %llu "
  4104. "parent %llu root %llu owner %llu offset %llu\n",
  4105. (unsigned long long)bytenr,
  4106. (unsigned long long)parent,
  4107. (unsigned long long)root_objectid,
  4108. (unsigned long long)owner_objectid,
  4109. (unsigned long long)owner_offset);
  4110. }
  4111. leaf = path->nodes[0];
  4112. item_size = btrfs_item_size_nr(leaf, extent_slot);
  4113. #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
  4114. if (item_size < sizeof(*ei)) {
  4115. BUG_ON(found_extent || extent_slot != path->slots[0]);
  4116. ret = convert_extent_item_v0(trans, extent_root, path,
  4117. owner_objectid, 0);
  4118. BUG_ON(ret < 0);
  4119. btrfs_release_path(path);
  4120. path->leave_spinning = 1;
  4121. key.objectid = bytenr;
  4122. key.type = BTRFS_EXTENT_ITEM_KEY;
  4123. key.offset = num_bytes;
  4124. ret = btrfs_search_slot(trans, extent_root, &key, path,
  4125. -1, 1);
  4126. if (ret) {
  4127. printk(KERN_ERR "umm, got %d back from search"
  4128. ", was looking for %llu\n", ret,
  4129. (unsigned long long)bytenr);
  4130. btrfs_print_leaf(extent_root, path->nodes[0]);
  4131. }
  4132. BUG_ON(ret);
  4133. extent_slot = path->slots[0];
  4134. leaf = path->nodes[0];
  4135. item_size = btrfs_item_size_nr(leaf, extent_slot);
  4136. }
  4137. #endif
  4138. BUG_ON(item_size < sizeof(*ei));
  4139. ei = btrfs_item_ptr(leaf, extent_slot,
  4140. struct btrfs_extent_item);
  4141. if (owner_objectid < BTRFS_FIRST_FREE_OBJECTID) {
  4142. struct btrfs_tree_block_info *bi;
  4143. BUG_ON(item_size < sizeof(*ei) + sizeof(*bi));
  4144. bi = (struct btrfs_tree_block_info *)(ei + 1);
  4145. WARN_ON(owner_objectid != btrfs_tree_block_level(leaf, bi));
  4146. }
  4147. refs = btrfs_extent_refs(leaf, ei);
  4148. BUG_ON(refs < refs_to_drop);
  4149. refs -= refs_to_drop;
  4150. if (refs > 0) {
  4151. if (extent_op)
  4152. __run_delayed_extent_op(extent_op, leaf, ei);
  4153. /*
  4154. * In the case of inline back ref, reference count will
  4155. * be updated by remove_extent_backref
  4156. */
  4157. if (iref) {
  4158. BUG_ON(!found_extent);
  4159. } else {
  4160. btrfs_set_extent_refs(leaf, ei, refs);
  4161. btrfs_mark_buffer_dirty(leaf);
  4162. }
  4163. if (found_extent) {
  4164. ret = remove_extent_backref(trans, extent_root, path,
  4165. iref, refs_to_drop,
  4166. is_data);
  4167. BUG_ON(ret);
  4168. }
  4169. } else {
  4170. if (found_extent) {
  4171. BUG_ON(is_data && refs_to_drop !=
  4172. extent_data_ref_count(root, path, iref));
  4173. if (iref) {
  4174. BUG_ON(path->slots[0] != extent_slot);
  4175. } else {
  4176. BUG_ON(path->slots[0] != extent_slot + 1);
  4177. path->slots[0] = extent_slot;
  4178. num_to_del = 2;
  4179. }
  4180. }
  4181. ret = btrfs_del_items(trans, extent_root, path, path->slots[0],
  4182. num_to_del);
  4183. BUG_ON(ret);
  4184. btrfs_release_path(path);
  4185. if (is_data) {
  4186. ret = btrfs_del_csums(trans, root, bytenr, num_bytes);
  4187. BUG_ON(ret);
  4188. } else {
  4189. invalidate_mapping_pages(info->btree_inode->i_mapping,
  4190. bytenr >> PAGE_CACHE_SHIFT,
  4191. (bytenr + num_bytes - 1) >> PAGE_CACHE_SHIFT);
  4192. }
  4193. ret = update_block_group(trans, root, bytenr, num_bytes, 0);
  4194. BUG_ON(ret);
  4195. }
  4196. btrfs_free_path(path);
  4197. return ret;
  4198. }
  4199. /*
  4200. * when we free an block, it is possible (and likely) that we free the last
  4201. * delayed ref for that extent as well. This searches the delayed ref tree for
  4202. * a given extent, and if there are no other delayed refs to be processed, it
  4203. * removes it from the tree.
  4204. */
  4205. static noinline int check_ref_cleanup(struct btrfs_trans_handle *trans,
  4206. struct btrfs_root *root, u64 bytenr)
  4207. {
  4208. struct btrfs_delayed_ref_head *head;
  4209. struct btrfs_delayed_ref_root *delayed_refs;
  4210. struct btrfs_delayed_ref_node *ref;
  4211. struct rb_node *node;
  4212. int ret = 0;
  4213. delayed_refs = &trans->transaction->delayed_refs;
  4214. spin_lock(&delayed_refs->lock);
  4215. head = btrfs_find_delayed_ref_head(trans, bytenr);
  4216. if (!head)
  4217. goto out;
  4218. node = rb_prev(&head->node.rb_node);
  4219. if (!node)
  4220. goto out;
  4221. ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);
  4222. /* there are still entries for this ref, we can't drop it */
  4223. if (ref->bytenr == bytenr)
  4224. goto out;
  4225. if (head->extent_op) {
  4226. if (!head->must_insert_reserved)
  4227. goto out;
  4228. kfree(head->extent_op);
  4229. head->extent_op = NULL;
  4230. }
  4231. /*
  4232. * waiting for the lock here would deadlock. If someone else has it
  4233. * locked they are already in the process of dropping it anyway
  4234. */
  4235. if (!mutex_trylock(&head->mutex))
  4236. goto out;
  4237. /*
  4238. * at this point we have a head with no other entries. Go
  4239. * ahead and process it.
  4240. */
  4241. head->node.in_tree = 0;
  4242. rb_erase(&head->node.rb_node, &delayed_refs->root);
  4243. delayed_refs->num_entries--;
  4244. /*
  4245. * we don't take a ref on the node because we're removing it from the
  4246. * tree, so we just steal the ref the tree was holding.
  4247. */
  4248. delayed_refs->num_heads--;
  4249. if (list_empty(&head->cluster))
  4250. delayed_refs->num_heads_ready--;
  4251. list_del_init(&head->cluster);
  4252. spin_unlock(&delayed_refs->lock);
  4253. BUG_ON(head->extent_op);
  4254. if (head->must_insert_reserved)
  4255. ret = 1;
  4256. mutex_unlock(&head->mutex);
  4257. btrfs_put_delayed_ref(&head->node);
  4258. return ret;
  4259. out:
  4260. spin_unlock(&delayed_refs->lock);
  4261. return 0;
  4262. }
  4263. void btrfs_free_tree_block(struct btrfs_trans_handle *trans,
  4264. struct btrfs_root *root,
  4265. struct extent_buffer *buf,
  4266. u64 parent, int last_ref)
  4267. {
  4268. struct btrfs_block_group_cache *cache = NULL;
  4269. int ret;
  4270. if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
  4271. ret = btrfs_add_delayed_tree_ref(trans, buf->start, buf->len,
  4272. parent, root->root_key.objectid,
  4273. btrfs_header_level(buf),
  4274. BTRFS_DROP_DELAYED_REF, NULL);
  4275. BUG_ON(ret);
  4276. }
  4277. if (!last_ref)
  4278. return;
  4279. cache = btrfs_lookup_block_group(root->fs_info, buf->start);
  4280. if (btrfs_header_generation(buf) == trans->transid) {
  4281. if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
  4282. ret = check_ref_cleanup(trans, root, buf->start);
  4283. if (!ret)
  4284. goto out;
  4285. }
  4286. if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) {
  4287. pin_down_extent(root, cache, buf->start, buf->len, 1);
  4288. goto out;
  4289. }
  4290. WARN_ON(test_bit(EXTENT_BUFFER_DIRTY, &buf->bflags));
  4291. btrfs_add_free_space(cache, buf->start, buf->len);
  4292. btrfs_update_reserved_bytes(cache, buf->len, RESERVE_FREE);
  4293. }
  4294. out:
  4295. /*
  4296. * Deleting the buffer, clear the corrupt flag since it doesn't matter
  4297. * anymore.
  4298. */
  4299. clear_bit(EXTENT_BUFFER_CORRUPT, &buf->bflags);
  4300. btrfs_put_block_group(cache);
  4301. }
  4302. int btrfs_free_extent(struct btrfs_trans_handle *trans,
  4303. struct btrfs_root *root,
  4304. u64 bytenr, u64 num_bytes, u64 parent,
  4305. u64 root_objectid, u64 owner, u64 offset)
  4306. {
  4307. int ret;
  4308. /*
  4309. * tree log blocks never actually go into the extent allocation
  4310. * tree, just update pinning info and exit early.
  4311. */
  4312. if (root_objectid == BTRFS_TREE_LOG_OBJECTID) {
  4313. WARN_ON(owner >= BTRFS_FIRST_FREE_OBJECTID);
  4314. /* unlocks the pinned mutex */
  4315. btrfs_pin_extent(root, bytenr, num_bytes, 1);
  4316. ret = 0;
  4317. } else if (owner < BTRFS_FIRST_FREE_OBJECTID) {
  4318. ret = btrfs_add_delayed_tree_ref(trans, bytenr, num_bytes,
  4319. parent, root_objectid, (int)owner,
  4320. BTRFS_DROP_DELAYED_REF, NULL);
  4321. BUG_ON(ret);
  4322. } else {
  4323. ret = btrfs_add_delayed_data_ref(trans, bytenr, num_bytes,
  4324. parent, root_objectid, owner,
  4325. offset, BTRFS_DROP_DELAYED_REF, NULL);
  4326. BUG_ON(ret);
  4327. }
  4328. return ret;
  4329. }
  4330. static u64 stripe_align(struct btrfs_root *root, u64 val)
  4331. {
  4332. u64 mask = ((u64)root->stripesize - 1);
  4333. u64 ret = (val + mask) & ~mask;
  4334. return ret;
  4335. }
  4336. /*
  4337. * when we wait for progress in the block group caching, its because
  4338. * our allocation attempt failed at least once. So, we must sleep
  4339. * and let some progress happen before we try again.
  4340. *
  4341. * This function will sleep at least once waiting for new free space to
  4342. * show up, and then it will check the block group free space numbers
  4343. * for our min num_bytes. Another option is to have it go ahead
  4344. * and look in the rbtree for a free extent of a given size, but this
  4345. * is a good start.
  4346. */
  4347. static noinline int
  4348. wait_block_group_cache_progress(struct btrfs_block_group_cache *cache,
  4349. u64 num_bytes)
  4350. {
  4351. struct btrfs_caching_control *caching_ctl;
  4352. DEFINE_WAIT(wait);
  4353. caching_ctl = get_caching_control(cache);
  4354. if (!caching_ctl)
  4355. return 0;
  4356. wait_event(caching_ctl->wait, block_group_cache_done(cache) ||
  4357. (cache->free_space_ctl->free_space >= num_bytes));
  4358. put_caching_control(caching_ctl);
  4359. return 0;
  4360. }
  4361. static noinline int
  4362. wait_block_group_cache_done(struct btrfs_block_group_cache *cache)
  4363. {
  4364. struct btrfs_caching_control *caching_ctl;
  4365. DEFINE_WAIT(wait);
  4366. caching_ctl = get_caching_control(cache);
  4367. if (!caching_ctl)
  4368. return 0;
  4369. wait_event(caching_ctl->wait, block_group_cache_done(cache));
  4370. put_caching_control(caching_ctl);
  4371. return 0;
  4372. }
  4373. static int get_block_group_index(struct btrfs_block_group_cache *cache)
  4374. {
  4375. int index;
  4376. if (cache->flags & BTRFS_BLOCK_GROUP_RAID10)
  4377. index = 0;
  4378. else if (cache->flags & BTRFS_BLOCK_GROUP_RAID1)
  4379. index = 1;
  4380. else if (cache->flags & BTRFS_BLOCK_GROUP_DUP)
  4381. index = 2;
  4382. else if (cache->flags & BTRFS_BLOCK_GROUP_RAID0)
  4383. index = 3;
  4384. else
  4385. index = 4;
  4386. return index;
  4387. }
  4388. enum btrfs_loop_type {
  4389. LOOP_FIND_IDEAL = 0,
  4390. LOOP_CACHING_NOWAIT = 1,
  4391. LOOP_CACHING_WAIT = 2,
  4392. LOOP_ALLOC_CHUNK = 3,
  4393. LOOP_NO_EMPTY_SIZE = 4,
  4394. };
  4395. /*
  4396. * walks the btree of allocated extents and find a hole of a given size.
  4397. * The key ins is changed to record the hole:
  4398. * ins->objectid == block start
  4399. * ins->flags = BTRFS_EXTENT_ITEM_KEY
  4400. * ins->offset == number of blocks
  4401. * Any available blocks before search_start are skipped.
  4402. */
  4403. static noinline int find_free_extent(struct btrfs_trans_handle *trans,
  4404. struct btrfs_root *orig_root,
  4405. u64 num_bytes, u64 empty_size,
  4406. u64 search_start, u64 search_end,
  4407. u64 hint_byte, struct btrfs_key *ins,
  4408. u64 data)
  4409. {
  4410. int ret = 0;
  4411. struct btrfs_root *root = orig_root->fs_info->extent_root;
  4412. struct btrfs_free_cluster *last_ptr = NULL;
  4413. struct btrfs_block_group_cache *block_group = NULL;
  4414. int empty_cluster = 2 * 1024 * 1024;
  4415. int allowed_chunk_alloc = 0;
  4416. int done_chunk_alloc = 0;
  4417. struct btrfs_space_info *space_info;
  4418. int last_ptr_loop = 0;
  4419. int loop = 0;
  4420. int index = 0;
  4421. int alloc_type = (data & BTRFS_BLOCK_GROUP_DATA) ?
  4422. RESERVE_ALLOC_NO_ACCOUNT : RESERVE_ALLOC;
  4423. bool found_uncached_bg = false;
  4424. bool failed_cluster_refill = false;
  4425. bool failed_alloc = false;
  4426. bool use_cluster = true;
  4427. bool have_caching_bg = false;
  4428. u64 ideal_cache_percent = 0;
  4429. u64 ideal_cache_offset = 0;
  4430. WARN_ON(num_bytes < root->sectorsize);
  4431. btrfs_set_key_type(ins, BTRFS_EXTENT_ITEM_KEY);
  4432. ins->objectid = 0;
  4433. ins->offset = 0;
  4434. space_info = __find_space_info(root->fs_info, data);
  4435. if (!space_info) {
  4436. printk(KERN_ERR "No space info for %llu\n", data);
  4437. return -ENOSPC;
  4438. }
  4439. /*
  4440. * If the space info is for both data and metadata it means we have a
  4441. * small filesystem and we can't use the clustering stuff.
  4442. */
  4443. if (btrfs_mixed_space_info(space_info))
  4444. use_cluster = false;
  4445. if (orig_root->ref_cows || empty_size)
  4446. allowed_chunk_alloc = 1;
  4447. if (data & BTRFS_BLOCK_GROUP_METADATA && use_cluster) {
  4448. last_ptr = &root->fs_info->meta_alloc_cluster;
  4449. if (!btrfs_test_opt(root, SSD))
  4450. empty_cluster = 64 * 1024;
  4451. }
  4452. if ((data & BTRFS_BLOCK_GROUP_DATA) && use_cluster &&
  4453. btrfs_test_opt(root, SSD)) {
  4454. last_ptr = &root->fs_info->data_alloc_cluster;
  4455. }
  4456. if (last_ptr) {
  4457. spin_lock(&last_ptr->lock);
  4458. if (last_ptr->block_group)
  4459. hint_byte = last_ptr->window_start;
  4460. spin_unlock(&last_ptr->lock);
  4461. }
  4462. search_start = max(search_start, first_logical_byte(root, 0));
  4463. search_start = max(search_start, hint_byte);
  4464. if (!last_ptr)
  4465. empty_cluster = 0;
  4466. if (search_start == hint_byte) {
  4467. ideal_cache:
  4468. block_group = btrfs_lookup_block_group(root->fs_info,
  4469. search_start);
  4470. /*
  4471. * we don't want to use the block group if it doesn't match our
  4472. * allocation bits, or if its not cached.
  4473. *
  4474. * However if we are re-searching with an ideal block group
  4475. * picked out then we don't care that the block group is cached.
  4476. */
  4477. if (block_group && block_group_bits(block_group, data) &&
  4478. (block_group->cached != BTRFS_CACHE_NO ||
  4479. search_start == ideal_cache_offset)) {
  4480. down_read(&space_info->groups_sem);
  4481. if (list_empty(&block_group->list) ||
  4482. block_group->ro) {
  4483. /*
  4484. * someone is removing this block group,
  4485. * we can't jump into the have_block_group
  4486. * target because our list pointers are not
  4487. * valid
  4488. */
  4489. btrfs_put_block_group(block_group);
  4490. up_read(&space_info->groups_sem);
  4491. } else {
  4492. index = get_block_group_index(block_group);
  4493. goto have_block_group;
  4494. }
  4495. } else if (block_group) {
  4496. btrfs_put_block_group(block_group);
  4497. }
  4498. }
  4499. search:
  4500. have_caching_bg = false;
  4501. down_read(&space_info->groups_sem);
  4502. list_for_each_entry(block_group, &space_info->block_groups[index],
  4503. list) {
  4504. u64 offset;
  4505. int cached;
  4506. btrfs_get_block_group(block_group);
  4507. search_start = block_group->key.objectid;
  4508. /*
  4509. * this can happen if we end up cycling through all the
  4510. * raid types, but we want to make sure we only allocate
  4511. * for the proper type.
  4512. */
  4513. if (!block_group_bits(block_group, data)) {
  4514. u64 extra = BTRFS_BLOCK_GROUP_DUP |
  4515. BTRFS_BLOCK_GROUP_RAID1 |
  4516. BTRFS_BLOCK_GROUP_RAID10;
  4517. /*
  4518. * if they asked for extra copies and this block group
  4519. * doesn't provide them, bail. This does allow us to
  4520. * fill raid0 from raid1.
  4521. */
  4522. if ((data & extra) && !(block_group->flags & extra))
  4523. goto loop;
  4524. }
  4525. have_block_group:
  4526. if (unlikely(block_group->cached == BTRFS_CACHE_NO)) {
  4527. u64 free_percent;
  4528. ret = cache_block_group(block_group, trans,
  4529. orig_root, 1);
  4530. if (block_group->cached == BTRFS_CACHE_FINISHED)
  4531. goto have_block_group;
  4532. free_percent = btrfs_block_group_used(&block_group->item);
  4533. free_percent *= 100;
  4534. free_percent = div64_u64(free_percent,
  4535. block_group->key.offset);
  4536. free_percent = 100 - free_percent;
  4537. if (free_percent > ideal_cache_percent &&
  4538. likely(!block_group->ro)) {
  4539. ideal_cache_offset = block_group->key.objectid;
  4540. ideal_cache_percent = free_percent;
  4541. }
  4542. /*
  4543. * The caching workers are limited to 2 threads, so we
  4544. * can queue as much work as we care to.
  4545. */
  4546. if (loop > LOOP_FIND_IDEAL) {
  4547. ret = cache_block_group(block_group, trans,
  4548. orig_root, 0);
  4549. BUG_ON(ret);
  4550. }
  4551. found_uncached_bg = true;
  4552. /*
  4553. * If loop is set for cached only, try the next block
  4554. * group.
  4555. */
  4556. if (loop == LOOP_FIND_IDEAL)
  4557. goto loop;
  4558. }
  4559. cached = block_group_cache_done(block_group);
  4560. if (unlikely(!cached))
  4561. found_uncached_bg = true;
  4562. if (unlikely(block_group->ro))
  4563. goto loop;
  4564. spin_lock(&block_group->free_space_ctl->tree_lock);
  4565. if (cached &&
  4566. block_group->free_space_ctl->free_space <
  4567. num_bytes + empty_size) {
  4568. spin_unlock(&block_group->free_space_ctl->tree_lock);
  4569. goto loop;
  4570. }
  4571. spin_unlock(&block_group->free_space_ctl->tree_lock);
  4572. /*
  4573. * Ok we want to try and use the cluster allocator, so lets look
  4574. * there, unless we are on LOOP_NO_EMPTY_SIZE, since we will
  4575. * have tried the cluster allocator plenty of times at this
  4576. * point and not have found anything, so we are likely way too
  4577. * fragmented for the clustering stuff to find anything, so lets
  4578. * just skip it and let the allocator find whatever block it can
  4579. * find
  4580. */
  4581. if (last_ptr && loop < LOOP_NO_EMPTY_SIZE) {
  4582. /*
  4583. * the refill lock keeps out other
  4584. * people trying to start a new cluster
  4585. */
  4586. spin_lock(&last_ptr->refill_lock);
  4587. if (last_ptr->block_group &&
  4588. (last_ptr->block_group->ro ||
  4589. !block_group_bits(last_ptr->block_group, data))) {
  4590. offset = 0;
  4591. goto refill_cluster;
  4592. }
  4593. offset = btrfs_alloc_from_cluster(block_group, last_ptr,
  4594. num_bytes, search_start);
  4595. if (offset) {
  4596. /* we have a block, we're done */
  4597. spin_unlock(&last_ptr->refill_lock);
  4598. goto checks;
  4599. }
  4600. spin_lock(&last_ptr->lock);
  4601. /*
  4602. * whoops, this cluster doesn't actually point to
  4603. * this block group. Get a ref on the block
  4604. * group is does point to and try again
  4605. */
  4606. if (!last_ptr_loop && last_ptr->block_group &&
  4607. last_ptr->block_group != block_group &&
  4608. index <=
  4609. get_block_group_index(last_ptr->block_group)) {
  4610. btrfs_put_block_group(block_group);
  4611. block_group = last_ptr->block_group;
  4612. btrfs_get_block_group(block_group);
  4613. spin_unlock(&last_ptr->lock);
  4614. spin_unlock(&last_ptr->refill_lock);
  4615. last_ptr_loop = 1;
  4616. search_start = block_group->key.objectid;
  4617. /*
  4618. * we know this block group is properly
  4619. * in the list because
  4620. * btrfs_remove_block_group, drops the
  4621. * cluster before it removes the block
  4622. * group from the list
  4623. */
  4624. goto have_block_group;
  4625. }
  4626. spin_unlock(&last_ptr->lock);
  4627. refill_cluster:
  4628. /*
  4629. * this cluster didn't work out, free it and
  4630. * start over
  4631. */
  4632. btrfs_return_cluster_to_free_space(NULL, last_ptr);
  4633. last_ptr_loop = 0;
  4634. /* allocate a cluster in this block group */
  4635. ret = btrfs_find_space_cluster(trans, root,
  4636. block_group, last_ptr,
  4637. offset, num_bytes,
  4638. empty_cluster + empty_size);
  4639. if (ret == 0) {
  4640. /*
  4641. * now pull our allocation out of this
  4642. * cluster
  4643. */
  4644. offset = btrfs_alloc_from_cluster(block_group,
  4645. last_ptr, num_bytes,
  4646. search_start);
  4647. if (offset) {
  4648. /* we found one, proceed */
  4649. spin_unlock(&last_ptr->refill_lock);
  4650. goto checks;
  4651. }
  4652. } else if (!cached && loop > LOOP_CACHING_NOWAIT
  4653. && !failed_cluster_refill) {
  4654. spin_unlock(&last_ptr->refill_lock);
  4655. failed_cluster_refill = true;
  4656. wait_block_group_cache_progress(block_group,
  4657. num_bytes + empty_cluster + empty_size);
  4658. goto have_block_group;
  4659. }
  4660. /*
  4661. * at this point we either didn't find a cluster
  4662. * or we weren't able to allocate a block from our
  4663. * cluster. Free the cluster we've been trying
  4664. * to use, and go to the next block group
  4665. */
  4666. btrfs_return_cluster_to_free_space(NULL, last_ptr);
  4667. spin_unlock(&last_ptr->refill_lock);
  4668. goto loop;
  4669. }
  4670. offset = btrfs_find_space_for_alloc(block_group, search_start,
  4671. num_bytes, empty_size);
  4672. /*
  4673. * If we didn't find a chunk, and we haven't failed on this
  4674. * block group before, and this block group is in the middle of
  4675. * caching and we are ok with waiting, then go ahead and wait
  4676. * for progress to be made, and set failed_alloc to true.
  4677. *
  4678. * If failed_alloc is true then we've already waited on this
  4679. * block group once and should move on to the next block group.
  4680. */
  4681. if (!offset && !failed_alloc && !cached &&
  4682. loop > LOOP_CACHING_NOWAIT) {
  4683. wait_block_group_cache_progress(block_group,
  4684. num_bytes + empty_size);
  4685. failed_alloc = true;
  4686. goto have_block_group;
  4687. } else if (!offset) {
  4688. if (!cached)
  4689. have_caching_bg = true;
  4690. goto loop;
  4691. }
  4692. checks:
  4693. search_start = stripe_align(root, offset);
  4694. /* move on to the next group */
  4695. if (search_start + num_bytes >= search_end) {
  4696. btrfs_add_free_space(block_group, offset, num_bytes);
  4697. goto loop;
  4698. }
  4699. /* move on to the next group */
  4700. if (search_start + num_bytes >
  4701. block_group->key.objectid + block_group->key.offset) {
  4702. btrfs_add_free_space(block_group, offset, num_bytes);
  4703. goto loop;
  4704. }
  4705. ins->objectid = search_start;
  4706. ins->offset = num_bytes;
  4707. if (offset < search_start)
  4708. btrfs_add_free_space(block_group, offset,
  4709. search_start - offset);
  4710. BUG_ON(offset > search_start);
  4711. ret = btrfs_update_reserved_bytes(block_group, num_bytes,
  4712. alloc_type);
  4713. if (ret == -EAGAIN) {
  4714. btrfs_add_free_space(block_group, offset, num_bytes);
  4715. goto loop;
  4716. }
  4717. /* we are all good, lets return */
  4718. ins->objectid = search_start;
  4719. ins->offset = num_bytes;
  4720. if (offset < search_start)
  4721. btrfs_add_free_space(block_group, offset,
  4722. search_start - offset);
  4723. BUG_ON(offset > search_start);
  4724. btrfs_put_block_group(block_group);
  4725. break;
  4726. loop:
  4727. failed_cluster_refill = false;
  4728. failed_alloc = false;
  4729. BUG_ON(index != get_block_group_index(block_group));
  4730. btrfs_put_block_group(block_group);
  4731. }
  4732. up_read(&space_info->groups_sem);
  4733. if (!ins->objectid && loop >= LOOP_CACHING_WAIT && have_caching_bg)
  4734. goto search;
  4735. if (!ins->objectid && ++index < BTRFS_NR_RAID_TYPES)
  4736. goto search;
  4737. /* LOOP_FIND_IDEAL, only search caching/cached bg's, and don't wait for
  4738. * for them to make caching progress. Also
  4739. * determine the best possible bg to cache
  4740. * LOOP_CACHING_NOWAIT, search partially cached block groups, kicking
  4741. * caching kthreads as we move along
  4742. * LOOP_CACHING_WAIT, search everything, and wait if our bg is caching
  4743. * LOOP_ALLOC_CHUNK, force a chunk allocation and try again
  4744. * LOOP_NO_EMPTY_SIZE, set empty_size and empty_cluster to 0 and try
  4745. * again
  4746. */
  4747. if (!ins->objectid && loop < LOOP_NO_EMPTY_SIZE) {
  4748. index = 0;
  4749. if (loop == LOOP_FIND_IDEAL && found_uncached_bg) {
  4750. found_uncached_bg = false;
  4751. loop++;
  4752. if (!ideal_cache_percent)
  4753. goto search;
  4754. /*
  4755. * 1 of the following 2 things have happened so far
  4756. *
  4757. * 1) We found an ideal block group for caching that
  4758. * is mostly full and will cache quickly, so we might
  4759. * as well wait for it.
  4760. *
  4761. * 2) We searched for cached only and we didn't find
  4762. * anything, and we didn't start any caching kthreads
  4763. * either, so chances are we will loop through and
  4764. * start a couple caching kthreads, and then come back
  4765. * around and just wait for them. This will be slower
  4766. * because we will have 2 caching kthreads reading at
  4767. * the same time when we could have just started one
  4768. * and waited for it to get far enough to give us an
  4769. * allocation, so go ahead and go to the wait caching
  4770. * loop.
  4771. */
  4772. loop = LOOP_CACHING_WAIT;
  4773. search_start = ideal_cache_offset;
  4774. ideal_cache_percent = 0;
  4775. goto ideal_cache;
  4776. } else if (loop == LOOP_FIND_IDEAL) {
  4777. /*
  4778. * Didn't find a uncached bg, wait on anything we find
  4779. * next.
  4780. */
  4781. loop = LOOP_CACHING_WAIT;
  4782. goto search;
  4783. }
  4784. loop++;
  4785. if (loop == LOOP_ALLOC_CHUNK) {
  4786. if (allowed_chunk_alloc) {
  4787. ret = do_chunk_alloc(trans, root, num_bytes +
  4788. 2 * 1024 * 1024, data,
  4789. CHUNK_ALLOC_LIMITED);
  4790. allowed_chunk_alloc = 0;
  4791. if (ret == 1)
  4792. done_chunk_alloc = 1;
  4793. } else if (!done_chunk_alloc &&
  4794. space_info->force_alloc ==
  4795. CHUNK_ALLOC_NO_FORCE) {
  4796. space_info->force_alloc = CHUNK_ALLOC_LIMITED;
  4797. }
  4798. /*
  4799. * We didn't allocate a chunk, go ahead and drop the
  4800. * empty size and loop again.
  4801. */
  4802. if (!done_chunk_alloc)
  4803. loop = LOOP_NO_EMPTY_SIZE;
  4804. }
  4805. if (loop == LOOP_NO_EMPTY_SIZE) {
  4806. empty_size = 0;
  4807. empty_cluster = 0;
  4808. }
  4809. goto search;
  4810. } else if (!ins->objectid) {
  4811. ret = -ENOSPC;
  4812. } else if (ins->objectid) {
  4813. ret = 0;
  4814. }
  4815. return ret;
  4816. }
  4817. static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
  4818. int dump_block_groups)
  4819. {
  4820. struct btrfs_block_group_cache *cache;
  4821. int index = 0;
  4822. spin_lock(&info->lock);
  4823. printk(KERN_INFO "space_info %llu has %llu free, is %sfull\n",
  4824. (unsigned long long)info->flags,
  4825. (unsigned long long)(info->total_bytes - info->bytes_used -
  4826. info->bytes_pinned - info->bytes_reserved -
  4827. info->bytes_readonly),
  4828. (info->full) ? "" : "not ");
  4829. printk(KERN_INFO "space_info total=%llu, used=%llu, pinned=%llu, "
  4830. "reserved=%llu, may_use=%llu, readonly=%llu\n",
  4831. (unsigned long long)info->total_bytes,
  4832. (unsigned long long)info->bytes_used,
  4833. (unsigned long long)info->bytes_pinned,
  4834. (unsigned long long)info->bytes_reserved,
  4835. (unsigned long long)info->bytes_may_use,
  4836. (unsigned long long)info->bytes_readonly);
  4837. spin_unlock(&info->lock);
  4838. if (!dump_block_groups)
  4839. return;
  4840. down_read(&info->groups_sem);
  4841. again:
  4842. list_for_each_entry(cache, &info->block_groups[index], list) {
  4843. spin_lock(&cache->lock);
  4844. printk(KERN_INFO "block group %llu has %llu bytes, %llu used "
  4845. "%llu pinned %llu reserved\n",
  4846. (unsigned long long)cache->key.objectid,
  4847. (unsigned long long)cache->key.offset,
  4848. (unsigned long long)btrfs_block_group_used(&cache->item),
  4849. (unsigned long long)cache->pinned,
  4850. (unsigned long long)cache->reserved);
  4851. btrfs_dump_free_space(cache, bytes);
  4852. spin_unlock(&cache->lock);
  4853. }
  4854. if (++index < BTRFS_NR_RAID_TYPES)
  4855. goto again;
  4856. up_read(&info->groups_sem);
  4857. }
  4858. int btrfs_reserve_extent(struct btrfs_trans_handle *trans,
  4859. struct btrfs_root *root,
  4860. u64 num_bytes, u64 min_alloc_size,
  4861. u64 empty_size, u64 hint_byte,
  4862. u64 search_end, struct btrfs_key *ins,
  4863. u64 data)
  4864. {
  4865. int ret;
  4866. u64 search_start = 0;
  4867. data = btrfs_get_alloc_profile(root, data);
  4868. again:
  4869. /*
  4870. * the only place that sets empty_size is btrfs_realloc_node, which
  4871. * is not called recursively on allocations
  4872. */
  4873. if (empty_size || root->ref_cows)
  4874. ret = do_chunk_alloc(trans, root->fs_info->extent_root,
  4875. num_bytes + 2 * 1024 * 1024, data,
  4876. CHUNK_ALLOC_NO_FORCE);
  4877. WARN_ON(num_bytes < root->sectorsize);
  4878. ret = find_free_extent(trans, root, num_bytes, empty_size,
  4879. search_start, search_end, hint_byte,
  4880. ins, data);
  4881. if (ret == -ENOSPC && num_bytes > min_alloc_size) {
  4882. num_bytes = num_bytes >> 1;
  4883. num_bytes = num_bytes & ~(root->sectorsize - 1);
  4884. num_bytes = max(num_bytes, min_alloc_size);
  4885. do_chunk_alloc(trans, root->fs_info->extent_root,
  4886. num_bytes, data, CHUNK_ALLOC_FORCE);
  4887. goto again;
  4888. }
  4889. if (ret == -ENOSPC && btrfs_test_opt(root, ENOSPC_DEBUG)) {
  4890. struct btrfs_space_info *sinfo;
  4891. sinfo = __find_space_info(root->fs_info, data);
  4892. printk(KERN_ERR "btrfs allocation failed flags %llu, "
  4893. "wanted %llu\n", (unsigned long long)data,
  4894. (unsigned long long)num_bytes);
  4895. dump_space_info(sinfo, num_bytes, 1);
  4896. }
  4897. trace_btrfs_reserved_extent_alloc(root, ins->objectid, ins->offset);
  4898. return ret;
  4899. }
  4900. static int __btrfs_free_reserved_extent(struct btrfs_root *root,
  4901. u64 start, u64 len, int pin)
  4902. {
  4903. struct btrfs_block_group_cache *cache;
  4904. int ret = 0;
  4905. cache = btrfs_lookup_block_group(root->fs_info, start);
  4906. if (!cache) {
  4907. printk(KERN_ERR "Unable to find block group for %llu\n",
  4908. (unsigned long long)start);
  4909. return -ENOSPC;
  4910. }
  4911. if (btrfs_test_opt(root, DISCARD))
  4912. ret = btrfs_discard_extent(root, start, len, NULL);
  4913. if (pin)
  4914. pin_down_extent(root, cache, start, len, 1);
  4915. else {
  4916. btrfs_add_free_space(cache, start, len);
  4917. btrfs_update_reserved_bytes(cache, len, RESERVE_FREE);
  4918. }
  4919. btrfs_put_block_group(cache);
  4920. trace_btrfs_reserved_extent_free(root, start, len);
  4921. return ret;
  4922. }
  4923. int btrfs_free_reserved_extent(struct btrfs_root *root,
  4924. u64 start, u64 len)
  4925. {
  4926. return __btrfs_free_reserved_extent(root, start, len, 0);
  4927. }
  4928. int btrfs_free_and_pin_reserved_extent(struct btrfs_root *root,
  4929. u64 start, u64 len)
  4930. {
  4931. return __btrfs_free_reserved_extent(root, start, len, 1);
  4932. }
  4933. static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
  4934. struct btrfs_root *root,
  4935. u64 parent, u64 root_objectid,
  4936. u64 flags, u64 owner, u64 offset,
  4937. struct btrfs_key *ins, int ref_mod)
  4938. {
  4939. int ret;
  4940. struct btrfs_fs_info *fs_info = root->fs_info;
  4941. struct btrfs_extent_item *extent_item;
  4942. struct btrfs_extent_inline_ref *iref;
  4943. struct btrfs_path *path;
  4944. struct extent_buffer *leaf;
  4945. int type;
  4946. u32 size;
  4947. if (parent > 0)
  4948. type = BTRFS_SHARED_DATA_REF_KEY;
  4949. else
  4950. type = BTRFS_EXTENT_DATA_REF_KEY;
  4951. size = sizeof(*extent_item) + btrfs_extent_inline_ref_size(type);
  4952. path = btrfs_alloc_path();
  4953. if (!path)
  4954. return -ENOMEM;
  4955. path->leave_spinning = 1;
  4956. ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
  4957. ins, size);
  4958. BUG_ON(ret);
  4959. leaf = path->nodes[0];
  4960. extent_item = btrfs_item_ptr(leaf, path->slots[0],
  4961. struct btrfs_extent_item);
  4962. btrfs_set_extent_refs(leaf, extent_item, ref_mod);
  4963. btrfs_set_extent_generation(leaf, extent_item, trans->transid);
  4964. btrfs_set_extent_flags(leaf, extent_item,
  4965. flags | BTRFS_EXTENT_FLAG_DATA);
  4966. iref = (struct btrfs_extent_inline_ref *)(extent_item + 1);
  4967. btrfs_set_extent_inline_ref_type(leaf, iref, type);
  4968. if (parent > 0) {
  4969. struct btrfs_shared_data_ref *ref;
  4970. ref = (struct btrfs_shared_data_ref *)(iref + 1);
  4971. btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
  4972. btrfs_set_shared_data_ref_count(leaf, ref, ref_mod);
  4973. } else {
  4974. struct btrfs_extent_data_ref *ref;
  4975. ref = (struct btrfs_extent_data_ref *)(&iref->offset);
  4976. btrfs_set_extent_data_ref_root(leaf, ref, root_objectid);
  4977. btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
  4978. btrfs_set_extent_data_ref_offset(leaf, ref, offset);
  4979. btrfs_set_extent_data_ref_count(leaf, ref, ref_mod);
  4980. }
  4981. btrfs_mark_buffer_dirty(path->nodes[0]);
  4982. btrfs_free_path(path);
  4983. ret = update_block_group(trans, root, ins->objectid, ins->offset, 1);
  4984. if (ret) {
  4985. printk(KERN_ERR "btrfs update block group failed for %llu "
  4986. "%llu\n", (unsigned long long)ins->objectid,
  4987. (unsigned long long)ins->offset);
  4988. BUG();
  4989. }
  4990. return ret;
  4991. }
  4992. static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
  4993. struct btrfs_root *root,
  4994. u64 parent, u64 root_objectid,
  4995. u64 flags, struct btrfs_disk_key *key,
  4996. int level, struct btrfs_key *ins)
  4997. {
  4998. int ret;
  4999. struct btrfs_fs_info *fs_info = root->fs_info;
  5000. struct btrfs_extent_item *extent_item;
  5001. struct btrfs_tree_block_info *block_info;
  5002. struct btrfs_extent_inline_ref *iref;
  5003. struct btrfs_path *path;
  5004. struct extent_buffer *leaf;
  5005. u32 size = sizeof(*extent_item) + sizeof(*block_info) + sizeof(*iref);
  5006. path = btrfs_alloc_path();
  5007. if (!path)
  5008. return -ENOMEM;
  5009. path->leave_spinning = 1;
  5010. ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
  5011. ins, size);
  5012. BUG_ON(ret);
  5013. leaf = path->nodes[0];
  5014. extent_item = btrfs_item_ptr(leaf, path->slots[0],
  5015. struct btrfs_extent_item);
  5016. btrfs_set_extent_refs(leaf, extent_item, 1);
  5017. btrfs_set_extent_generation(leaf, extent_item, trans->transid);
  5018. btrfs_set_extent_flags(leaf, extent_item,
  5019. flags | BTRFS_EXTENT_FLAG_TREE_BLOCK);
  5020. block_info = (struct btrfs_tree_block_info *)(extent_item + 1);
  5021. btrfs_set_tree_block_key(leaf, block_info, key);
  5022. btrfs_set_tree_block_level(leaf, block_info, level);
  5023. iref = (struct btrfs_extent_inline_ref *)(block_info + 1);
  5024. if (parent > 0) {
  5025. BUG_ON(!(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF));
  5026. btrfs_set_extent_inline_ref_type(leaf, iref,
  5027. BTRFS_SHARED_BLOCK_REF_KEY);
  5028. btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
  5029. } else {
  5030. btrfs_set_extent_inline_ref_type(leaf, iref,
  5031. BTRFS_TREE_BLOCK_REF_KEY);
  5032. btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
  5033. }
  5034. btrfs_mark_buffer_dirty(leaf);
  5035. btrfs_free_path(path);
  5036. ret = update_block_group(trans, root, ins->objectid, ins->offset, 1);
  5037. if (ret) {
  5038. printk(KERN_ERR "btrfs update block group failed for %llu "
  5039. "%llu\n", (unsigned long long)ins->objectid,
  5040. (unsigned long long)ins->offset);
  5041. BUG();
  5042. }
  5043. return ret;
  5044. }
  5045. int btrfs_alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
  5046. struct btrfs_root *root,
  5047. u64 root_objectid, u64 owner,
  5048. u64 offset, struct btrfs_key *ins)
  5049. {
  5050. int ret;
  5051. BUG_ON(root_objectid == BTRFS_TREE_LOG_OBJECTID);
  5052. ret = btrfs_add_delayed_data_ref(trans, ins->objectid, ins->offset,
  5053. 0, root_objectid, owner, offset,
  5054. BTRFS_ADD_DELAYED_EXTENT, NULL);
  5055. return ret;
  5056. }
  5057. /*
  5058. * this is used by the tree logging recovery code. It records that
  5059. * an extent has been allocated and makes sure to clear the free
  5060. * space cache bits as well
  5061. */
  5062. int btrfs_alloc_logged_file_extent(struct btrfs_trans_handle *trans,
  5063. struct btrfs_root *root,
  5064. u64 root_objectid, u64 owner, u64 offset,
  5065. struct btrfs_key *ins)
  5066. {
  5067. int ret;
  5068. struct btrfs_block_group_cache *block_group;
  5069. struct btrfs_caching_control *caching_ctl;
  5070. u64 start = ins->objectid;
  5071. u64 num_bytes = ins->offset;
  5072. block_group = btrfs_lookup_block_group(root->fs_info, ins->objectid);
  5073. cache_block_group(block_group, trans, NULL, 0);
  5074. caching_ctl = get_caching_control(block_group);
  5075. if (!caching_ctl) {
  5076. BUG_ON(!block_group_cache_done(block_group));
  5077. ret = btrfs_remove_free_space(block_group, start, num_bytes);
  5078. BUG_ON(ret);
  5079. } else {
  5080. mutex_lock(&caching_ctl->mutex);
  5081. if (start >= caching_ctl->progress) {
  5082. ret = add_excluded_extent(root, start, num_bytes);
  5083. BUG_ON(ret);
  5084. } else if (start + num_bytes <= caching_ctl->progress) {
  5085. ret = btrfs_remove_free_space(block_group,
  5086. start, num_bytes);
  5087. BUG_ON(ret);
  5088. } else {
  5089. num_bytes = caching_ctl->progress - start;
  5090. ret = btrfs_remove_free_space(block_group,
  5091. start, num_bytes);
  5092. BUG_ON(ret);
  5093. start = caching_ctl->progress;
  5094. num_bytes = ins->objectid + ins->offset -
  5095. caching_ctl->progress;
  5096. ret = add_excluded_extent(root, start, num_bytes);
  5097. BUG_ON(ret);
  5098. }
  5099. mutex_unlock(&caching_ctl->mutex);
  5100. put_caching_control(caching_ctl);
  5101. }
  5102. ret = btrfs_update_reserved_bytes(block_group, ins->offset,
  5103. RESERVE_ALLOC_NO_ACCOUNT);
  5104. BUG_ON(ret);
  5105. btrfs_put_block_group(block_group);
  5106. ret = alloc_reserved_file_extent(trans, root, 0, root_objectid,
  5107. 0, owner, offset, ins, 1);
  5108. return ret;
  5109. }
  5110. struct extent_buffer *btrfs_init_new_buffer(struct btrfs_trans_handle *trans,
  5111. struct btrfs_root *root,
  5112. u64 bytenr, u32 blocksize,
  5113. int level)
  5114. {
  5115. struct extent_buffer *buf;
  5116. buf = btrfs_find_create_tree_block(root, bytenr, blocksize);
  5117. if (!buf)
  5118. return ERR_PTR(-ENOMEM);
  5119. btrfs_set_header_generation(buf, trans->transid);
  5120. btrfs_set_buffer_lockdep_class(root->root_key.objectid, buf, level);
  5121. btrfs_tree_lock(buf);
  5122. clean_tree_block(trans, root, buf);
  5123. btrfs_set_lock_blocking(buf);
  5124. btrfs_set_buffer_uptodate(buf);
  5125. if (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID) {
  5126. /*
  5127. * we allow two log transactions at a time, use different
  5128. * EXENT bit to differentiate dirty pages.
  5129. */
  5130. if (root->log_transid % 2 == 0)
  5131. set_extent_dirty(&root->dirty_log_pages, buf->start,
  5132. buf->start + buf->len - 1, GFP_NOFS);
  5133. else
  5134. set_extent_new(&root->dirty_log_pages, buf->start,
  5135. buf->start + buf->len - 1, GFP_NOFS);
  5136. } else {
  5137. set_extent_dirty(&trans->transaction->dirty_pages, buf->start,
  5138. buf->start + buf->len - 1, GFP_NOFS);
  5139. }
  5140. trans->blocks_used++;
  5141. /* this returns a buffer locked for blocking */
  5142. return buf;
  5143. }
  5144. static struct btrfs_block_rsv *
  5145. use_block_rsv(struct btrfs_trans_handle *trans,
  5146. struct btrfs_root *root, u32 blocksize)
  5147. {
  5148. struct btrfs_block_rsv *block_rsv;
  5149. struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
  5150. int ret;
  5151. block_rsv = get_block_rsv(trans, root);
  5152. if (block_rsv->size == 0) {
  5153. ret = reserve_metadata_bytes(root, block_rsv, blocksize, 0);
  5154. /*
  5155. * If we couldn't reserve metadata bytes try and use some from
  5156. * the global reserve.
  5157. */
  5158. if (ret && block_rsv != global_rsv) {
  5159. ret = block_rsv_use_bytes(global_rsv, blocksize);
  5160. if (!ret)
  5161. return global_rsv;
  5162. return ERR_PTR(ret);
  5163. } else if (ret) {
  5164. return ERR_PTR(ret);
  5165. }
  5166. return block_rsv;
  5167. }
  5168. ret = block_rsv_use_bytes(block_rsv, blocksize);
  5169. if (!ret)
  5170. return block_rsv;
  5171. if (ret) {
  5172. static DEFINE_RATELIMIT_STATE(_rs,
  5173. DEFAULT_RATELIMIT_INTERVAL,
  5174. /*DEFAULT_RATELIMIT_BURST*/ 2);
  5175. if (__ratelimit(&_rs)) {
  5176. printk(KERN_DEBUG "btrfs: block rsv returned %d\n", ret);
  5177. WARN_ON(1);
  5178. }
  5179. ret = reserve_metadata_bytes(root, block_rsv, blocksize, 0);
  5180. if (!ret) {
  5181. return block_rsv;
  5182. } else if (ret && block_rsv != global_rsv) {
  5183. ret = block_rsv_use_bytes(global_rsv, blocksize);
  5184. if (!ret)
  5185. return global_rsv;
  5186. }
  5187. }
  5188. return ERR_PTR(-ENOSPC);
  5189. }
  5190. static void unuse_block_rsv(struct btrfs_block_rsv *block_rsv, u32 blocksize)
  5191. {
  5192. block_rsv_add_bytes(block_rsv, blocksize, 0);
  5193. block_rsv_release_bytes(block_rsv, NULL, 0);
  5194. }
  5195. /*
  5196. * finds a free extent and does all the dirty work required for allocation
  5197. * returns the key for the extent through ins, and a tree buffer for
  5198. * the first block of the extent through buf.
  5199. *
  5200. * returns the tree buffer or NULL.
  5201. */
  5202. struct extent_buffer *btrfs_alloc_free_block(struct btrfs_trans_handle *trans,
  5203. struct btrfs_root *root, u32 blocksize,
  5204. u64 parent, u64 root_objectid,
  5205. struct btrfs_disk_key *key, int level,
  5206. u64 hint, u64 empty_size)
  5207. {
  5208. struct btrfs_key ins;
  5209. struct btrfs_block_rsv *block_rsv;
  5210. struct extent_buffer *buf;
  5211. u64 flags = 0;
  5212. int ret;
  5213. block_rsv = use_block_rsv(trans, root, blocksize);
  5214. if (IS_ERR(block_rsv))
  5215. return ERR_CAST(block_rsv);
  5216. ret = btrfs_reserve_extent(trans, root, blocksize, blocksize,
  5217. empty_size, hint, (u64)-1, &ins, 0);
  5218. if (ret) {
  5219. unuse_block_rsv(block_rsv, blocksize);
  5220. return ERR_PTR(ret);
  5221. }
  5222. buf = btrfs_init_new_buffer(trans, root, ins.objectid,
  5223. blocksize, level);
  5224. BUG_ON(IS_ERR(buf));
  5225. if (root_objectid == BTRFS_TREE_RELOC_OBJECTID) {
  5226. if (parent == 0)
  5227. parent = ins.objectid;
  5228. flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF;
  5229. } else
  5230. BUG_ON(parent > 0);
  5231. if (root_objectid != BTRFS_TREE_LOG_OBJECTID) {
  5232. struct btrfs_delayed_extent_op *extent_op;
  5233. extent_op = kmalloc(sizeof(*extent_op), GFP_NOFS);
  5234. BUG_ON(!extent_op);
  5235. if (key)
  5236. memcpy(&extent_op->key, key, sizeof(extent_op->key));
  5237. else
  5238. memset(&extent_op->key, 0, sizeof(extent_op->key));
  5239. extent_op->flags_to_set = flags;
  5240. extent_op->update_key = 1;
  5241. extent_op->update_flags = 1;
  5242. extent_op->is_data = 0;
  5243. ret = btrfs_add_delayed_tree_ref(trans, ins.objectid,
  5244. ins.offset, parent, root_objectid,
  5245. level, BTRFS_ADD_DELAYED_EXTENT,
  5246. extent_op);
  5247. BUG_ON(ret);
  5248. }
  5249. return buf;
  5250. }
  5251. struct walk_control {
  5252. u64 refs[BTRFS_MAX_LEVEL];
  5253. u64 flags[BTRFS_MAX_LEVEL];
  5254. struct btrfs_key update_progress;
  5255. int stage;
  5256. int level;
  5257. int shared_level;
  5258. int update_ref;
  5259. int keep_locks;
  5260. int reada_slot;
  5261. int reada_count;
  5262. };
  5263. #define DROP_REFERENCE 1
  5264. #define UPDATE_BACKREF 2
  5265. static noinline void reada_walk_down(struct btrfs_trans_handle *trans,
  5266. struct btrfs_root *root,
  5267. struct walk_control *wc,
  5268. struct btrfs_path *path)
  5269. {
  5270. u64 bytenr;
  5271. u64 generation;
  5272. u64 refs;
  5273. u64 flags;
  5274. u32 nritems;
  5275. u32 blocksize;
  5276. struct btrfs_key key;
  5277. struct extent_buffer *eb;
  5278. int ret;
  5279. int slot;
  5280. int nread = 0;
  5281. if (path->slots[wc->level] < wc->reada_slot) {
  5282. wc->reada_count = wc->reada_count * 2 / 3;
  5283. wc->reada_count = max(wc->reada_count, 2);
  5284. } else {
  5285. wc->reada_count = wc->reada_count * 3 / 2;
  5286. wc->reada_count = min_t(int, wc->reada_count,
  5287. BTRFS_NODEPTRS_PER_BLOCK(root));
  5288. }
  5289. eb = path->nodes[wc->level];
  5290. nritems = btrfs_header_nritems(eb);
  5291. blocksize = btrfs_level_size(root, wc->level - 1);
  5292. for (slot = path->slots[wc->level]; slot < nritems; slot++) {
  5293. if (nread >= wc->reada_count)
  5294. break;
  5295. cond_resched();
  5296. bytenr = btrfs_node_blockptr(eb, slot);
  5297. generation = btrfs_node_ptr_generation(eb, slot);
  5298. if (slot == path->slots[wc->level])
  5299. goto reada;
  5300. if (wc->stage == UPDATE_BACKREF &&
  5301. generation <= root->root_key.offset)
  5302. continue;
  5303. /* We don't lock the tree block, it's OK to be racy here */
  5304. ret = btrfs_lookup_extent_info(trans, root, bytenr, blocksize,
  5305. &refs, &flags);
  5306. BUG_ON(ret);
  5307. BUG_ON(refs == 0);
  5308. if (wc->stage == DROP_REFERENCE) {
  5309. if (refs == 1)
  5310. goto reada;
  5311. if (wc->level == 1 &&
  5312. (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
  5313. continue;
  5314. if (!wc->update_ref ||
  5315. generation <= root->root_key.offset)
  5316. continue;
  5317. btrfs_node_key_to_cpu(eb, &key, slot);
  5318. ret = btrfs_comp_cpu_keys(&key,
  5319. &wc->update_progress);
  5320. if (ret < 0)
  5321. continue;
  5322. } else {
  5323. if (wc->level == 1 &&
  5324. (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
  5325. continue;
  5326. }
  5327. reada:
  5328. ret = readahead_tree_block(root, bytenr, blocksize,
  5329. generation);
  5330. if (ret)
  5331. break;
  5332. nread++;
  5333. }
  5334. wc->reada_slot = slot;
  5335. }
  5336. /*
  5337. * hepler to process tree block while walking down the tree.
  5338. *
  5339. * when wc->stage == UPDATE_BACKREF, this function updates
  5340. * back refs for pointers in the block.
  5341. *
  5342. * NOTE: return value 1 means we should stop walking down.
  5343. */
  5344. static noinline int walk_down_proc(struct btrfs_trans_handle *trans,
  5345. struct btrfs_root *root,
  5346. struct btrfs_path *path,
  5347. struct walk_control *wc, int lookup_info)
  5348. {
  5349. int level = wc->level;
  5350. struct extent_buffer *eb = path->nodes[level];
  5351. u64 flag = BTRFS_BLOCK_FLAG_FULL_BACKREF;
  5352. int ret;
  5353. if (wc->stage == UPDATE_BACKREF &&
  5354. btrfs_header_owner(eb) != root->root_key.objectid)
  5355. return 1;
  5356. /*
  5357. * when reference count of tree block is 1, it won't increase
  5358. * again. once full backref flag is set, we never clear it.
  5359. */
  5360. if (lookup_info &&
  5361. ((wc->stage == DROP_REFERENCE && wc->refs[level] != 1) ||
  5362. (wc->stage == UPDATE_BACKREF && !(wc->flags[level] & flag)))) {
  5363. BUG_ON(!path->locks[level]);
  5364. ret = btrfs_lookup_extent_info(trans, root,
  5365. eb->start, eb->len,
  5366. &wc->refs[level],
  5367. &wc->flags[level]);
  5368. BUG_ON(ret);
  5369. BUG_ON(wc->refs[level] == 0);
  5370. }
  5371. if (wc->stage == DROP_REFERENCE) {
  5372. if (wc->refs[level] > 1)
  5373. return 1;
  5374. if (path->locks[level] && !wc->keep_locks) {
  5375. btrfs_tree_unlock_rw(eb, path->locks[level]);
  5376. path->locks[level] = 0;
  5377. }
  5378. return 0;
  5379. }
  5380. /* wc->stage == UPDATE_BACKREF */
  5381. if (!(wc->flags[level] & flag)) {
  5382. BUG_ON(!path->locks[level]);
  5383. ret = btrfs_inc_ref(trans, root, eb, 1);
  5384. BUG_ON(ret);
  5385. ret = btrfs_dec_ref(trans, root, eb, 0);
  5386. BUG_ON(ret);
  5387. ret = btrfs_set_disk_extent_flags(trans, root, eb->start,
  5388. eb->len, flag, 0);
  5389. BUG_ON(ret);
  5390. wc->flags[level] |= flag;
  5391. }
  5392. /*
  5393. * the block is shared by multiple trees, so it's not good to
  5394. * keep the tree lock
  5395. */
  5396. if (path->locks[level] && level > 0) {
  5397. btrfs_tree_unlock_rw(eb, path->locks[level]);
  5398. path->locks[level] = 0;
  5399. }
  5400. return 0;
  5401. }
  5402. /*
  5403. * hepler to process tree block pointer.
  5404. *
  5405. * when wc->stage == DROP_REFERENCE, this function checks
  5406. * reference count of the block pointed to. if the block
  5407. * is shared and we need update back refs for the subtree
  5408. * rooted at the block, this function changes wc->stage to
  5409. * UPDATE_BACKREF. if the block is shared and there is no
  5410. * need to update back, this function drops the reference
  5411. * to the block.
  5412. *
  5413. * NOTE: return value 1 means we should stop walking down.
  5414. */
  5415. static noinline int do_walk_down(struct btrfs_trans_handle *trans,
  5416. struct btrfs_root *root,
  5417. struct btrfs_path *path,
  5418. struct walk_control *wc, int *lookup_info)
  5419. {
  5420. u64 bytenr;
  5421. u64 generation;
  5422. u64 parent;
  5423. u32 blocksize;
  5424. struct btrfs_key key;
  5425. struct extent_buffer *next;
  5426. int level = wc->level;
  5427. int reada = 0;
  5428. int ret = 0;
  5429. generation = btrfs_node_ptr_generation(path->nodes[level],
  5430. path->slots[level]);
  5431. /*
  5432. * if the lower level block was created before the snapshot
  5433. * was created, we know there is no need to update back refs
  5434. * for the subtree
  5435. */
  5436. if (wc->stage == UPDATE_BACKREF &&
  5437. generation <= root->root_key.offset) {
  5438. *lookup_info = 1;
  5439. return 1;
  5440. }
  5441. bytenr = btrfs_node_blockptr(path->nodes[level], path->slots[level]);
  5442. blocksize = btrfs_level_size(root, level - 1);
  5443. next = btrfs_find_tree_block(root, bytenr, blocksize);
  5444. if (!next) {
  5445. next = btrfs_find_create_tree_block(root, bytenr, blocksize);
  5446. if (!next)
  5447. return -ENOMEM;
  5448. reada = 1;
  5449. }
  5450. btrfs_tree_lock(next);
  5451. btrfs_set_lock_blocking(next);
  5452. ret = btrfs_lookup_extent_info(trans, root, bytenr, blocksize,
  5453. &wc->refs[level - 1],
  5454. &wc->flags[level - 1]);
  5455. BUG_ON(ret);
  5456. BUG_ON(wc->refs[level - 1] == 0);
  5457. *lookup_info = 0;
  5458. if (wc->stage == DROP_REFERENCE) {
  5459. if (wc->refs[level - 1] > 1) {
  5460. if (level == 1 &&
  5461. (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
  5462. goto skip;
  5463. if (!wc->update_ref ||
  5464. generation <= root->root_key.offset)
  5465. goto skip;
  5466. btrfs_node_key_to_cpu(path->nodes[level], &key,
  5467. path->slots[level]);
  5468. ret = btrfs_comp_cpu_keys(&key, &wc->update_progress);
  5469. if (ret < 0)
  5470. goto skip;
  5471. wc->stage = UPDATE_BACKREF;
  5472. wc->shared_level = level - 1;
  5473. }
  5474. } else {
  5475. if (level == 1 &&
  5476. (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
  5477. goto skip;
  5478. }
  5479. if (!btrfs_buffer_uptodate(next, generation)) {
  5480. btrfs_tree_unlock(next);
  5481. free_extent_buffer(next);
  5482. next = NULL;
  5483. *lookup_info = 1;
  5484. }
  5485. if (!next) {
  5486. if (reada && level == 1)
  5487. reada_walk_down(trans, root, wc, path);
  5488. next = read_tree_block(root, bytenr, blocksize, generation);
  5489. if (!next)
  5490. return -EIO;
  5491. btrfs_tree_lock(next);
  5492. btrfs_set_lock_blocking(next);
  5493. }
  5494. level--;
  5495. BUG_ON(level != btrfs_header_level(next));
  5496. path->nodes[level] = next;
  5497. path->slots[level] = 0;
  5498. path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
  5499. wc->level = level;
  5500. if (wc->level == 1)
  5501. wc->reada_slot = 0;
  5502. return 0;
  5503. skip:
  5504. wc->refs[level - 1] = 0;
  5505. wc->flags[level - 1] = 0;
  5506. if (wc->stage == DROP_REFERENCE) {
  5507. if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
  5508. parent = path->nodes[level]->start;
  5509. } else {
  5510. BUG_ON(root->root_key.objectid !=
  5511. btrfs_header_owner(path->nodes[level]));
  5512. parent = 0;
  5513. }
  5514. ret = btrfs_free_extent(trans, root, bytenr, blocksize, parent,
  5515. root->root_key.objectid, level - 1, 0);
  5516. BUG_ON(ret);
  5517. }
  5518. btrfs_tree_unlock(next);
  5519. free_extent_buffer(next);
  5520. *lookup_info = 1;
  5521. return 1;
  5522. }
  5523. /*
  5524. * hepler to process tree block while walking up the tree.
  5525. *
  5526. * when wc->stage == DROP_REFERENCE, this function drops
  5527. * reference count on the block.
  5528. *
  5529. * when wc->stage == UPDATE_BACKREF, this function changes
  5530. * wc->stage back to DROP_REFERENCE if we changed wc->stage
  5531. * to UPDATE_BACKREF previously while processing the block.
  5532. *
  5533. * NOTE: return value 1 means we should stop walking up.
  5534. */
  5535. static noinline int walk_up_proc(struct btrfs_trans_handle *trans,
  5536. struct btrfs_root *root,
  5537. struct btrfs_path *path,
  5538. struct walk_control *wc)
  5539. {
  5540. int ret;
  5541. int level = wc->level;
  5542. struct extent_buffer *eb = path->nodes[level];
  5543. u64 parent = 0;
  5544. if (wc->stage == UPDATE_BACKREF) {
  5545. BUG_ON(wc->shared_level < level);
  5546. if (level < wc->shared_level)
  5547. goto out;
  5548. ret = find_next_key(path, level + 1, &wc->update_progress);
  5549. if (ret > 0)
  5550. wc->update_ref = 0;
  5551. wc->stage = DROP_REFERENCE;
  5552. wc->shared_level = -1;
  5553. path->slots[level] = 0;
  5554. /*
  5555. * check reference count again if the block isn't locked.
  5556. * we should start walking down the tree again if reference
  5557. * count is one.
  5558. */
  5559. if (!path->locks[level]) {
  5560. BUG_ON(level == 0);
  5561. btrfs_tree_lock(eb);
  5562. btrfs_set_lock_blocking(eb);
  5563. path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
  5564. ret = btrfs_lookup_extent_info(trans, root,
  5565. eb->start, eb->len,
  5566. &wc->refs[level],
  5567. &wc->flags[level]);
  5568. BUG_ON(ret);
  5569. BUG_ON(wc->refs[level] == 0);
  5570. if (wc->refs[level] == 1) {
  5571. btrfs_tree_unlock_rw(eb, path->locks[level]);
  5572. return 1;
  5573. }
  5574. }
  5575. }
  5576. /* wc->stage == DROP_REFERENCE */
  5577. BUG_ON(wc->refs[level] > 1 && !path->locks[level]);
  5578. if (wc->refs[level] == 1) {
  5579. if (level == 0) {
  5580. if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
  5581. ret = btrfs_dec_ref(trans, root, eb, 1);
  5582. else
  5583. ret = btrfs_dec_ref(trans, root, eb, 0);
  5584. BUG_ON(ret);
  5585. }
  5586. /* make block locked assertion in clean_tree_block happy */
  5587. if (!path->locks[level] &&
  5588. btrfs_header_generation(eb) == trans->transid) {
  5589. btrfs_tree_lock(eb);
  5590. btrfs_set_lock_blocking(eb);
  5591. path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
  5592. }
  5593. clean_tree_block(trans, root, eb);
  5594. }
  5595. if (eb == root->node) {
  5596. if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
  5597. parent = eb->start;
  5598. else
  5599. BUG_ON(root->root_key.objectid !=
  5600. btrfs_header_owner(eb));
  5601. } else {
  5602. if (wc->flags[level + 1] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
  5603. parent = path->nodes[level + 1]->start;
  5604. else
  5605. BUG_ON(root->root_key.objectid !=
  5606. btrfs_header_owner(path->nodes[level + 1]));
  5607. }
  5608. btrfs_free_tree_block(trans, root, eb, parent, wc->refs[level] == 1);
  5609. out:
  5610. wc->refs[level] = 0;
  5611. wc->flags[level] = 0;
  5612. return 0;
  5613. }
  5614. static noinline int walk_down_tree(struct btrfs_trans_handle *trans,
  5615. struct btrfs_root *root,
  5616. struct btrfs_path *path,
  5617. struct walk_control *wc)
  5618. {
  5619. int level = wc->level;
  5620. int lookup_info = 1;
  5621. int ret;
  5622. while (level >= 0) {
  5623. ret = walk_down_proc(trans, root, path, wc, lookup_info);
  5624. if (ret > 0)
  5625. break;
  5626. if (level == 0)
  5627. break;
  5628. if (path->slots[level] >=
  5629. btrfs_header_nritems(path->nodes[level]))
  5630. break;
  5631. ret = do_walk_down(trans, root, path, wc, &lookup_info);
  5632. if (ret > 0) {
  5633. path->slots[level]++;
  5634. continue;
  5635. } else if (ret < 0)
  5636. return ret;
  5637. level = wc->level;
  5638. }
  5639. return 0;
  5640. }
  5641. static noinline int walk_up_tree(struct btrfs_trans_handle *trans,
  5642. struct btrfs_root *root,
  5643. struct btrfs_path *path,
  5644. struct walk_control *wc, int max_level)
  5645. {
  5646. int level = wc->level;
  5647. int ret;
  5648. path->slots[level] = btrfs_header_nritems(path->nodes[level]);
  5649. while (level < max_level && path->nodes[level]) {
  5650. wc->level = level;
  5651. if (path->slots[level] + 1 <
  5652. btrfs_header_nritems(path->nodes[level])) {
  5653. path->slots[level]++;
  5654. return 0;
  5655. } else {
  5656. ret = walk_up_proc(trans, root, path, wc);
  5657. if (ret > 0)
  5658. return 0;
  5659. if (path->locks[level]) {
  5660. btrfs_tree_unlock_rw(path->nodes[level],
  5661. path->locks[level]);
  5662. path->locks[level] = 0;
  5663. }
  5664. free_extent_buffer(path->nodes[level]);
  5665. path->nodes[level] = NULL;
  5666. level++;
  5667. }
  5668. }
  5669. return 1;
  5670. }
  5671. /*
  5672. * drop a subvolume tree.
  5673. *
  5674. * this function traverses the tree freeing any blocks that only
  5675. * referenced by the tree.
  5676. *
  5677. * when a shared tree block is found. this function decreases its
  5678. * reference count by one. if update_ref is true, this function
  5679. * also make sure backrefs for the shared block and all lower level
  5680. * blocks are properly updated.
  5681. */
  5682. void btrfs_drop_snapshot(struct btrfs_root *root,
  5683. struct btrfs_block_rsv *block_rsv, int update_ref)
  5684. {
  5685. struct btrfs_path *path;
  5686. struct btrfs_trans_handle *trans;
  5687. struct btrfs_root *tree_root = root->fs_info->tree_root;
  5688. struct btrfs_root_item *root_item = &root->root_item;
  5689. struct walk_control *wc;
  5690. struct btrfs_key key;
  5691. int err = 0;
  5692. int ret;
  5693. int level;
  5694. path = btrfs_alloc_path();
  5695. if (!path) {
  5696. err = -ENOMEM;
  5697. goto out;
  5698. }
  5699. wc = kzalloc(sizeof(*wc), GFP_NOFS);
  5700. if (!wc) {
  5701. btrfs_free_path(path);
  5702. err = -ENOMEM;
  5703. goto out;
  5704. }
  5705. trans = btrfs_start_transaction(tree_root, 0);
  5706. BUG_ON(IS_ERR(trans));
  5707. if (block_rsv)
  5708. trans->block_rsv = block_rsv;
  5709. if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) {
  5710. level = btrfs_header_level(root->node);
  5711. path->nodes[level] = btrfs_lock_root_node(root);
  5712. btrfs_set_lock_blocking(path->nodes[level]);
  5713. path->slots[level] = 0;
  5714. path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
  5715. memset(&wc->update_progress, 0,
  5716. sizeof(wc->update_progress));
  5717. } else {
  5718. btrfs_disk_key_to_cpu(&key, &root_item->drop_progress);
  5719. memcpy(&wc->update_progress, &key,
  5720. sizeof(wc->update_progress));
  5721. level = root_item->drop_level;
  5722. BUG_ON(level == 0);
  5723. path->lowest_level = level;
  5724. ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
  5725. path->lowest_level = 0;
  5726. if (ret < 0) {
  5727. err = ret;
  5728. goto out_free;
  5729. }
  5730. WARN_ON(ret > 0);
  5731. /*
  5732. * unlock our path, this is safe because only this
  5733. * function is allowed to delete this snapshot
  5734. */
  5735. btrfs_unlock_up_safe(path, 0);
  5736. level = btrfs_header_level(root->node);
  5737. while (1) {
  5738. btrfs_tree_lock(path->nodes[level]);
  5739. btrfs_set_lock_blocking(path->nodes[level]);
  5740. ret = btrfs_lookup_extent_info(trans, root,
  5741. path->nodes[level]->start,
  5742. path->nodes[level]->len,
  5743. &wc->refs[level],
  5744. &wc->flags[level]);
  5745. BUG_ON(ret);
  5746. BUG_ON(wc->refs[level] == 0);
  5747. if (level == root_item->drop_level)
  5748. break;
  5749. btrfs_tree_unlock(path->nodes[level]);
  5750. WARN_ON(wc->refs[level] != 1);
  5751. level--;
  5752. }
  5753. }
  5754. wc->level = level;
  5755. wc->shared_level = -1;
  5756. wc->stage = DROP_REFERENCE;
  5757. wc->update_ref = update_ref;
  5758. wc->keep_locks = 0;
  5759. wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
  5760. while (1) {
  5761. ret = walk_down_tree(trans, root, path, wc);
  5762. if (ret < 0) {
  5763. err = ret;
  5764. break;
  5765. }
  5766. ret = walk_up_tree(trans, root, path, wc, BTRFS_MAX_LEVEL);
  5767. if (ret < 0) {
  5768. err = ret;
  5769. break;
  5770. }
  5771. if (ret > 0) {
  5772. BUG_ON(wc->stage != DROP_REFERENCE);
  5773. break;
  5774. }
  5775. if (wc->stage == DROP_REFERENCE) {
  5776. level = wc->level;
  5777. btrfs_node_key(path->nodes[level],
  5778. &root_item->drop_progress,
  5779. path->slots[level]);
  5780. root_item->drop_level = level;
  5781. }
  5782. BUG_ON(wc->level == 0);
  5783. if (btrfs_should_end_transaction(trans, tree_root)) {
  5784. ret = btrfs_update_root(trans, tree_root,
  5785. &root->root_key,
  5786. root_item);
  5787. BUG_ON(ret);
  5788. btrfs_end_transaction_throttle(trans, tree_root);
  5789. trans = btrfs_start_transaction(tree_root, 0);
  5790. BUG_ON(IS_ERR(trans));
  5791. if (block_rsv)
  5792. trans->block_rsv = block_rsv;
  5793. }
  5794. }
  5795. btrfs_release_path(path);
  5796. BUG_ON(err);
  5797. ret = btrfs_del_root(trans, tree_root, &root->root_key);
  5798. BUG_ON(ret);
  5799. if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) {
  5800. ret = btrfs_find_last_root(tree_root, root->root_key.objectid,
  5801. NULL, NULL);
  5802. BUG_ON(ret < 0);
  5803. if (ret > 0) {
  5804. /* if we fail to delete the orphan item this time
  5805. * around, it'll get picked up the next time.
  5806. *
  5807. * The most common failure here is just -ENOENT.
  5808. */
  5809. btrfs_del_orphan_item(trans, tree_root,
  5810. root->root_key.objectid);
  5811. }
  5812. }
  5813. if (root->in_radix) {
  5814. btrfs_free_fs_root(tree_root->fs_info, root);
  5815. } else {
  5816. free_extent_buffer(root->node);
  5817. free_extent_buffer(root->commit_root);
  5818. kfree(root);
  5819. }
  5820. out_free:
  5821. btrfs_end_transaction_throttle(trans, tree_root);
  5822. kfree(wc);
  5823. btrfs_free_path(path);
  5824. out:
  5825. if (err)
  5826. btrfs_std_error(root->fs_info, err);
  5827. return;
  5828. }
  5829. /*
  5830. * drop subtree rooted at tree block 'node'.
  5831. *
  5832. * NOTE: this function will unlock and release tree block 'node'
  5833. */
  5834. int btrfs_drop_subtree(struct btrfs_trans_handle *trans,
  5835. struct btrfs_root *root,
  5836. struct extent_buffer *node,
  5837. struct extent_buffer *parent)
  5838. {
  5839. struct btrfs_path *path;
  5840. struct walk_control *wc;
  5841. int level;
  5842. int parent_level;
  5843. int ret = 0;
  5844. int wret;
  5845. BUG_ON(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID);
  5846. path = btrfs_alloc_path();
  5847. if (!path)
  5848. return -ENOMEM;
  5849. wc = kzalloc(sizeof(*wc), GFP_NOFS);
  5850. if (!wc) {
  5851. btrfs_free_path(path);
  5852. return -ENOMEM;
  5853. }
  5854. btrfs_assert_tree_locked(parent);
  5855. parent_level = btrfs_header_level(parent);
  5856. extent_buffer_get(parent);
  5857. path->nodes[parent_level] = parent;
  5858. path->slots[parent_level] = btrfs_header_nritems(parent);
  5859. btrfs_assert_tree_locked(node);
  5860. level = btrfs_header_level(node);
  5861. path->nodes[level] = node;
  5862. path->slots[level] = 0;
  5863. path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
  5864. wc->refs[parent_level] = 1;
  5865. wc->flags[parent_level] = BTRFS_BLOCK_FLAG_FULL_BACKREF;
  5866. wc->level = level;
  5867. wc->shared_level = -1;
  5868. wc->stage = DROP_REFERENCE;
  5869. wc->update_ref = 0;
  5870. wc->keep_locks = 1;
  5871. wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
  5872. while (1) {
  5873. wret = walk_down_tree(trans, root, path, wc);
  5874. if (wret < 0) {
  5875. ret = wret;
  5876. break;
  5877. }
  5878. wret = walk_up_tree(trans, root, path, wc, parent_level);
  5879. if (wret < 0)
  5880. ret = wret;
  5881. if (wret != 0)
  5882. break;
  5883. }
  5884. kfree(wc);
  5885. btrfs_free_path(path);
  5886. return ret;
  5887. }
  5888. static u64 update_block_group_flags(struct btrfs_root *root, u64 flags)
  5889. {
  5890. u64 num_devices;
  5891. u64 stripped = BTRFS_BLOCK_GROUP_RAID0 |
  5892. BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10;
  5893. /*
  5894. * we add in the count of missing devices because we want
  5895. * to make sure that any RAID levels on a degraded FS
  5896. * continue to be honored.
  5897. */
  5898. num_devices = root->fs_info->fs_devices->rw_devices +
  5899. root->fs_info->fs_devices->missing_devices;
  5900. if (num_devices == 1) {
  5901. stripped |= BTRFS_BLOCK_GROUP_DUP;
  5902. stripped = flags & ~stripped;
  5903. /* turn raid0 into single device chunks */
  5904. if (flags & BTRFS_BLOCK_GROUP_RAID0)
  5905. return stripped;
  5906. /* turn mirroring into duplication */
  5907. if (flags & (BTRFS_BLOCK_GROUP_RAID1 |
  5908. BTRFS_BLOCK_GROUP_RAID10))
  5909. return stripped | BTRFS_BLOCK_GROUP_DUP;
  5910. return flags;
  5911. } else {
  5912. /* they already had raid on here, just return */
  5913. if (flags & stripped)
  5914. return flags;
  5915. stripped |= BTRFS_BLOCK_GROUP_DUP;
  5916. stripped = flags & ~stripped;
  5917. /* switch duplicated blocks with raid1 */
  5918. if (flags & BTRFS_BLOCK_GROUP_DUP)
  5919. return stripped | BTRFS_BLOCK_GROUP_RAID1;
  5920. /* turn single device chunks into raid0 */
  5921. return stripped | BTRFS_BLOCK_GROUP_RAID0;
  5922. }
  5923. return flags;
  5924. }
  5925. static int set_block_group_ro(struct btrfs_block_group_cache *cache, int force)
  5926. {
  5927. struct btrfs_space_info *sinfo = cache->space_info;
  5928. u64 num_bytes;
  5929. u64 min_allocable_bytes;
  5930. int ret = -ENOSPC;
  5931. /*
  5932. * We need some metadata space and system metadata space for
  5933. * allocating chunks in some corner cases until we force to set
  5934. * it to be readonly.
  5935. */
  5936. if ((sinfo->flags &
  5937. (BTRFS_BLOCK_GROUP_SYSTEM | BTRFS_BLOCK_GROUP_METADATA)) &&
  5938. !force)
  5939. min_allocable_bytes = 1 * 1024 * 1024;
  5940. else
  5941. min_allocable_bytes = 0;
  5942. spin_lock(&sinfo->lock);
  5943. spin_lock(&cache->lock);
  5944. if (cache->ro) {
  5945. ret = 0;
  5946. goto out;
  5947. }
  5948. num_bytes = cache->key.offset - cache->reserved - cache->pinned -
  5949. cache->bytes_super - btrfs_block_group_used(&cache->item);
  5950. if (sinfo->bytes_used + sinfo->bytes_reserved + sinfo->bytes_pinned +
  5951. sinfo->bytes_may_use + sinfo->bytes_readonly + num_bytes +
  5952. min_allocable_bytes <= sinfo->total_bytes) {
  5953. sinfo->bytes_readonly += num_bytes;
  5954. cache->ro = 1;
  5955. ret = 0;
  5956. }
  5957. out:
  5958. spin_unlock(&cache->lock);
  5959. spin_unlock(&sinfo->lock);
  5960. return ret;
  5961. }
  5962. int btrfs_set_block_group_ro(struct btrfs_root *root,
  5963. struct btrfs_block_group_cache *cache)
  5964. {
  5965. struct btrfs_trans_handle *trans;
  5966. u64 alloc_flags;
  5967. int ret;
  5968. BUG_ON(cache->ro);
  5969. trans = btrfs_join_transaction(root);
  5970. BUG_ON(IS_ERR(trans));
  5971. alloc_flags = update_block_group_flags(root, cache->flags);
  5972. if (alloc_flags != cache->flags)
  5973. do_chunk_alloc(trans, root, 2 * 1024 * 1024, alloc_flags,
  5974. CHUNK_ALLOC_FORCE);
  5975. ret = set_block_group_ro(cache, 0);
  5976. if (!ret)
  5977. goto out;
  5978. alloc_flags = get_alloc_profile(root, cache->space_info->flags);
  5979. ret = do_chunk_alloc(trans, root, 2 * 1024 * 1024, alloc_flags,
  5980. CHUNK_ALLOC_FORCE);
  5981. if (ret < 0)
  5982. goto out;
  5983. ret = set_block_group_ro(cache, 0);
  5984. out:
  5985. btrfs_end_transaction(trans, root);
  5986. return ret;
  5987. }
  5988. int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans,
  5989. struct btrfs_root *root, u64 type)
  5990. {
  5991. u64 alloc_flags = get_alloc_profile(root, type);
  5992. return do_chunk_alloc(trans, root, 2 * 1024 * 1024, alloc_flags,
  5993. CHUNK_ALLOC_FORCE);
  5994. }
  5995. /*
  5996. * helper to account the unused space of all the readonly block group in the
  5997. * list. takes mirrors into account.
  5998. */
  5999. static u64 __btrfs_get_ro_block_group_free_space(struct list_head *groups_list)
  6000. {
  6001. struct btrfs_block_group_cache *block_group;
  6002. u64 free_bytes = 0;
  6003. int factor;
  6004. list_for_each_entry(block_group, groups_list, list) {
  6005. spin_lock(&block_group->lock);
  6006. if (!block_group->ro) {
  6007. spin_unlock(&block_group->lock);
  6008. continue;
  6009. }
  6010. if (block_group->flags & (BTRFS_BLOCK_GROUP_RAID1 |
  6011. BTRFS_BLOCK_GROUP_RAID10 |
  6012. BTRFS_BLOCK_GROUP_DUP))
  6013. factor = 2;
  6014. else
  6015. factor = 1;
  6016. free_bytes += (block_group->key.offset -
  6017. btrfs_block_group_used(&block_group->item)) *
  6018. factor;
  6019. spin_unlock(&block_group->lock);
  6020. }
  6021. return free_bytes;
  6022. }
  6023. /*
  6024. * helper to account the unused space of all the readonly block group in the
  6025. * space_info. takes mirrors into account.
  6026. */
  6027. u64 btrfs_account_ro_block_groups_free_space(struct btrfs_space_info *sinfo)
  6028. {
  6029. int i;
  6030. u64 free_bytes = 0;
  6031. spin_lock(&sinfo->lock);
  6032. for(i = 0; i < BTRFS_NR_RAID_TYPES; i++)
  6033. if (!list_empty(&sinfo->block_groups[i]))
  6034. free_bytes += __btrfs_get_ro_block_group_free_space(
  6035. &sinfo->block_groups[i]);
  6036. spin_unlock(&sinfo->lock);
  6037. return free_bytes;
  6038. }
  6039. int btrfs_set_block_group_rw(struct btrfs_root *root,
  6040. struct btrfs_block_group_cache *cache)
  6041. {
  6042. struct btrfs_space_info *sinfo = cache->space_info;
  6043. u64 num_bytes;
  6044. BUG_ON(!cache->ro);
  6045. spin_lock(&sinfo->lock);
  6046. spin_lock(&cache->lock);
  6047. num_bytes = cache->key.offset - cache->reserved - cache->pinned -
  6048. cache->bytes_super - btrfs_block_group_used(&cache->item);
  6049. sinfo->bytes_readonly -= num_bytes;
  6050. cache->ro = 0;
  6051. spin_unlock(&cache->lock);
  6052. spin_unlock(&sinfo->lock);
  6053. return 0;
  6054. }
  6055. /*
  6056. * checks to see if its even possible to relocate this block group.
  6057. *
  6058. * @return - -1 if it's not a good idea to relocate this block group, 0 if its
  6059. * ok to go ahead and try.
  6060. */
  6061. int btrfs_can_relocate(struct btrfs_root *root, u64 bytenr)
  6062. {
  6063. struct btrfs_block_group_cache *block_group;
  6064. struct btrfs_space_info *space_info;
  6065. struct btrfs_fs_devices *fs_devices = root->fs_info->fs_devices;
  6066. struct btrfs_device *device;
  6067. u64 min_free;
  6068. u64 dev_min = 1;
  6069. u64 dev_nr = 0;
  6070. int index;
  6071. int full = 0;
  6072. int ret = 0;
  6073. block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
  6074. /* odd, couldn't find the block group, leave it alone */
  6075. if (!block_group)
  6076. return -1;
  6077. min_free = btrfs_block_group_used(&block_group->item);
  6078. /* no bytes used, we're good */
  6079. if (!min_free)
  6080. goto out;
  6081. space_info = block_group->space_info;
  6082. spin_lock(&space_info->lock);
  6083. full = space_info->full;
  6084. /*
  6085. * if this is the last block group we have in this space, we can't
  6086. * relocate it unless we're able to allocate a new chunk below.
  6087. *
  6088. * Otherwise, we need to make sure we have room in the space to handle
  6089. * all of the extents from this block group. If we can, we're good
  6090. */
  6091. if ((space_info->total_bytes != block_group->key.offset) &&
  6092. (space_info->bytes_used + space_info->bytes_reserved +
  6093. space_info->bytes_pinned + space_info->bytes_readonly +
  6094. min_free < space_info->total_bytes)) {
  6095. spin_unlock(&space_info->lock);
  6096. goto out;
  6097. }
  6098. spin_unlock(&space_info->lock);
  6099. /*
  6100. * ok we don't have enough space, but maybe we have free space on our
  6101. * devices to allocate new chunks for relocation, so loop through our
  6102. * alloc devices and guess if we have enough space. However, if we
  6103. * were marked as full, then we know there aren't enough chunks, and we
  6104. * can just return.
  6105. */
  6106. ret = -1;
  6107. if (full)
  6108. goto out;
  6109. /*
  6110. * index:
  6111. * 0: raid10
  6112. * 1: raid1
  6113. * 2: dup
  6114. * 3: raid0
  6115. * 4: single
  6116. */
  6117. index = get_block_group_index(block_group);
  6118. if (index == 0) {
  6119. dev_min = 4;
  6120. /* Divide by 2 */
  6121. min_free >>= 1;
  6122. } else if (index == 1) {
  6123. dev_min = 2;
  6124. } else if (index == 2) {
  6125. /* Multiply by 2 */
  6126. min_free <<= 1;
  6127. } else if (index == 3) {
  6128. dev_min = fs_devices->rw_devices;
  6129. do_div(min_free, dev_min);
  6130. }
  6131. mutex_lock(&root->fs_info->chunk_mutex);
  6132. list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
  6133. u64 dev_offset;
  6134. /*
  6135. * check to make sure we can actually find a chunk with enough
  6136. * space to fit our block group in.
  6137. */
  6138. if (device->total_bytes > device->bytes_used + min_free) {
  6139. ret = find_free_dev_extent(NULL, device, min_free,
  6140. &dev_offset, NULL);
  6141. if (!ret)
  6142. dev_nr++;
  6143. if (dev_nr >= dev_min)
  6144. break;
  6145. ret = -1;
  6146. }
  6147. }
  6148. mutex_unlock(&root->fs_info->chunk_mutex);
  6149. out:
  6150. btrfs_put_block_group(block_group);
  6151. return ret;
  6152. }
  6153. static int find_first_block_group(struct btrfs_root *root,
  6154. struct btrfs_path *path, struct btrfs_key *key)
  6155. {
  6156. int ret = 0;
  6157. struct btrfs_key found_key;
  6158. struct extent_buffer *leaf;
  6159. int slot;
  6160. ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
  6161. if (ret < 0)
  6162. goto out;
  6163. while (1) {
  6164. slot = path->slots[0];
  6165. leaf = path->nodes[0];
  6166. if (slot >= btrfs_header_nritems(leaf)) {
  6167. ret = btrfs_next_leaf(root, path);
  6168. if (ret == 0)
  6169. continue;
  6170. if (ret < 0)
  6171. goto out;
  6172. break;
  6173. }
  6174. btrfs_item_key_to_cpu(leaf, &found_key, slot);
  6175. if (found_key.objectid >= key->objectid &&
  6176. found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) {
  6177. ret = 0;
  6178. goto out;
  6179. }
  6180. path->slots[0]++;
  6181. }
  6182. out:
  6183. return ret;
  6184. }
  6185. void btrfs_put_block_group_cache(struct btrfs_fs_info *info)
  6186. {
  6187. struct btrfs_block_group_cache *block_group;
  6188. u64 last = 0;
  6189. while (1) {
  6190. struct inode *inode;
  6191. block_group = btrfs_lookup_first_block_group(info, last);
  6192. while (block_group) {
  6193. spin_lock(&block_group->lock);
  6194. if (block_group->iref)
  6195. break;
  6196. spin_unlock(&block_group->lock);
  6197. block_group = next_block_group(info->tree_root,
  6198. block_group);
  6199. }
  6200. if (!block_group) {
  6201. if (last == 0)
  6202. break;
  6203. last = 0;
  6204. continue;
  6205. }
  6206. inode = block_group->inode;
  6207. block_group->iref = 0;
  6208. block_group->inode = NULL;
  6209. spin_unlock(&block_group->lock);
  6210. iput(inode);
  6211. last = block_group->key.objectid + block_group->key.offset;
  6212. btrfs_put_block_group(block_group);
  6213. }
  6214. }
  6215. int btrfs_free_block_groups(struct btrfs_fs_info *info)
  6216. {
  6217. struct btrfs_block_group_cache *block_group;
  6218. struct btrfs_space_info *space_info;
  6219. struct btrfs_caching_control *caching_ctl;
  6220. struct rb_node *n;
  6221. down_write(&info->extent_commit_sem);
  6222. while (!list_empty(&info->caching_block_groups)) {
  6223. caching_ctl = list_entry(info->caching_block_groups.next,
  6224. struct btrfs_caching_control, list);
  6225. list_del(&caching_ctl->list);
  6226. put_caching_control(caching_ctl);
  6227. }
  6228. up_write(&info->extent_commit_sem);
  6229. spin_lock(&info->block_group_cache_lock);
  6230. while ((n = rb_last(&info->block_group_cache_tree)) != NULL) {
  6231. block_group = rb_entry(n, struct btrfs_block_group_cache,
  6232. cache_node);
  6233. rb_erase(&block_group->cache_node,
  6234. &info->block_group_cache_tree);
  6235. spin_unlock(&info->block_group_cache_lock);
  6236. down_write(&block_group->space_info->groups_sem);
  6237. list_del(&block_group->list);
  6238. up_write(&block_group->space_info->groups_sem);
  6239. if (block_group->cached == BTRFS_CACHE_STARTED)
  6240. wait_block_group_cache_done(block_group);
  6241. /*
  6242. * We haven't cached this block group, which means we could
  6243. * possibly have excluded extents on this block group.
  6244. */
  6245. if (block_group->cached == BTRFS_CACHE_NO)
  6246. free_excluded_extents(info->extent_root, block_group);
  6247. btrfs_remove_free_space_cache(block_group);
  6248. btrfs_put_block_group(block_group);
  6249. spin_lock(&info->block_group_cache_lock);
  6250. }
  6251. spin_unlock(&info->block_group_cache_lock);
  6252. /* now that all the block groups are freed, go through and
  6253. * free all the space_info structs. This is only called during
  6254. * the final stages of unmount, and so we know nobody is
  6255. * using them. We call synchronize_rcu() once before we start,
  6256. * just to be on the safe side.
  6257. */
  6258. synchronize_rcu();
  6259. release_global_block_rsv(info);
  6260. while(!list_empty(&info->space_info)) {
  6261. space_info = list_entry(info->space_info.next,
  6262. struct btrfs_space_info,
  6263. list);
  6264. if (space_info->bytes_pinned > 0 ||
  6265. space_info->bytes_reserved > 0 ||
  6266. space_info->bytes_may_use > 0) {
  6267. WARN_ON(1);
  6268. dump_space_info(space_info, 0, 0);
  6269. }
  6270. list_del(&space_info->list);
  6271. kfree(space_info);
  6272. }
  6273. return 0;
  6274. }
  6275. static void __link_block_group(struct btrfs_space_info *space_info,
  6276. struct btrfs_block_group_cache *cache)
  6277. {
  6278. int index = get_block_group_index(cache);
  6279. down_write(&space_info->groups_sem);
  6280. list_add_tail(&cache->list, &space_info->block_groups[index]);
  6281. up_write(&space_info->groups_sem);
  6282. }
  6283. int btrfs_read_block_groups(struct btrfs_root *root)
  6284. {
  6285. struct btrfs_path *path;
  6286. int ret;
  6287. struct btrfs_block_group_cache *cache;
  6288. struct btrfs_fs_info *info = root->fs_info;
  6289. struct btrfs_space_info *space_info;
  6290. struct btrfs_key key;
  6291. struct btrfs_key found_key;
  6292. struct extent_buffer *leaf;
  6293. int need_clear = 0;
  6294. u64 cache_gen;
  6295. root = info->extent_root;
  6296. key.objectid = 0;
  6297. key.offset = 0;
  6298. btrfs_set_key_type(&key, BTRFS_BLOCK_GROUP_ITEM_KEY);
  6299. path = btrfs_alloc_path();
  6300. if (!path)
  6301. return -ENOMEM;
  6302. path->reada = 1;
  6303. cache_gen = btrfs_super_cache_generation(root->fs_info->super_copy);
  6304. if (btrfs_test_opt(root, SPACE_CACHE) &&
  6305. btrfs_super_generation(root->fs_info->super_copy) != cache_gen)
  6306. need_clear = 1;
  6307. if (btrfs_test_opt(root, CLEAR_CACHE))
  6308. need_clear = 1;
  6309. while (1) {
  6310. ret = find_first_block_group(root, path, &key);
  6311. if (ret > 0)
  6312. break;
  6313. if (ret != 0)
  6314. goto error;
  6315. leaf = path->nodes[0];
  6316. btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
  6317. cache = kzalloc(sizeof(*cache), GFP_NOFS);
  6318. if (!cache) {
  6319. ret = -ENOMEM;
  6320. goto error;
  6321. }
  6322. cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
  6323. GFP_NOFS);
  6324. if (!cache->free_space_ctl) {
  6325. kfree(cache);
  6326. ret = -ENOMEM;
  6327. goto error;
  6328. }
  6329. atomic_set(&cache->count, 1);
  6330. spin_lock_init(&cache->lock);
  6331. cache->fs_info = info;
  6332. INIT_LIST_HEAD(&cache->list);
  6333. INIT_LIST_HEAD(&cache->cluster_list);
  6334. if (need_clear)
  6335. cache->disk_cache_state = BTRFS_DC_CLEAR;
  6336. read_extent_buffer(leaf, &cache->item,
  6337. btrfs_item_ptr_offset(leaf, path->slots[0]),
  6338. sizeof(cache->item));
  6339. memcpy(&cache->key, &found_key, sizeof(found_key));
  6340. key.objectid = found_key.objectid + found_key.offset;
  6341. btrfs_release_path(path);
  6342. cache->flags = btrfs_block_group_flags(&cache->item);
  6343. cache->sectorsize = root->sectorsize;
  6344. btrfs_init_free_space_ctl(cache);
  6345. /*
  6346. * We need to exclude the super stripes now so that the space
  6347. * info has super bytes accounted for, otherwise we'll think
  6348. * we have more space than we actually do.
  6349. */
  6350. exclude_super_stripes(root, cache);
  6351. /*
  6352. * check for two cases, either we are full, and therefore
  6353. * don't need to bother with the caching work since we won't
  6354. * find any space, or we are empty, and we can just add all
  6355. * the space in and be done with it. This saves us _alot_ of
  6356. * time, particularly in the full case.
  6357. */
  6358. if (found_key.offset == btrfs_block_group_used(&cache->item)) {
  6359. cache->last_byte_to_unpin = (u64)-1;
  6360. cache->cached = BTRFS_CACHE_FINISHED;
  6361. free_excluded_extents(root, cache);
  6362. } else if (btrfs_block_group_used(&cache->item) == 0) {
  6363. cache->last_byte_to_unpin = (u64)-1;
  6364. cache->cached = BTRFS_CACHE_FINISHED;
  6365. add_new_free_space(cache, root->fs_info,
  6366. found_key.objectid,
  6367. found_key.objectid +
  6368. found_key.offset);
  6369. free_excluded_extents(root, cache);
  6370. }
  6371. ret = update_space_info(info, cache->flags, found_key.offset,
  6372. btrfs_block_group_used(&cache->item),
  6373. &space_info);
  6374. BUG_ON(ret);
  6375. cache->space_info = space_info;
  6376. spin_lock(&cache->space_info->lock);
  6377. cache->space_info->bytes_readonly += cache->bytes_super;
  6378. spin_unlock(&cache->space_info->lock);
  6379. __link_block_group(space_info, cache);
  6380. ret = btrfs_add_block_group_cache(root->fs_info, cache);
  6381. BUG_ON(ret);
  6382. set_avail_alloc_bits(root->fs_info, cache->flags);
  6383. if (btrfs_chunk_readonly(root, cache->key.objectid))
  6384. set_block_group_ro(cache, 1);
  6385. }
  6386. list_for_each_entry_rcu(space_info, &root->fs_info->space_info, list) {
  6387. if (!(get_alloc_profile(root, space_info->flags) &
  6388. (BTRFS_BLOCK_GROUP_RAID10 |
  6389. BTRFS_BLOCK_GROUP_RAID1 |
  6390. BTRFS_BLOCK_GROUP_DUP)))
  6391. continue;
  6392. /*
  6393. * avoid allocating from un-mirrored block group if there are
  6394. * mirrored block groups.
  6395. */
  6396. list_for_each_entry(cache, &space_info->block_groups[3], list)
  6397. set_block_group_ro(cache, 1);
  6398. list_for_each_entry(cache, &space_info->block_groups[4], list)
  6399. set_block_group_ro(cache, 1);
  6400. }
  6401. init_global_block_rsv(info);
  6402. ret = 0;
  6403. error:
  6404. btrfs_free_path(path);
  6405. return ret;
  6406. }
  6407. int btrfs_make_block_group(struct btrfs_trans_handle *trans,
  6408. struct btrfs_root *root, u64 bytes_used,
  6409. u64 type, u64 chunk_objectid, u64 chunk_offset,
  6410. u64 size)
  6411. {
  6412. int ret;
  6413. struct btrfs_root *extent_root;
  6414. struct btrfs_block_group_cache *cache;
  6415. extent_root = root->fs_info->extent_root;
  6416. root->fs_info->last_trans_log_full_commit = trans->transid;
  6417. cache = kzalloc(sizeof(*cache), GFP_NOFS);
  6418. if (!cache)
  6419. return -ENOMEM;
  6420. cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
  6421. GFP_NOFS);
  6422. if (!cache->free_space_ctl) {
  6423. kfree(cache);
  6424. return -ENOMEM;
  6425. }
  6426. cache->key.objectid = chunk_offset;
  6427. cache->key.offset = size;
  6428. cache->key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
  6429. cache->sectorsize = root->sectorsize;
  6430. cache->fs_info = root->fs_info;
  6431. atomic_set(&cache->count, 1);
  6432. spin_lock_init(&cache->lock);
  6433. INIT_LIST_HEAD(&cache->list);
  6434. INIT_LIST_HEAD(&cache->cluster_list);
  6435. btrfs_init_free_space_ctl(cache);
  6436. btrfs_set_block_group_used(&cache->item, bytes_used);
  6437. btrfs_set_block_group_chunk_objectid(&cache->item, chunk_objectid);
  6438. cache->flags = type;
  6439. btrfs_set_block_group_flags(&cache->item, type);
  6440. cache->last_byte_to_unpin = (u64)-1;
  6441. cache->cached = BTRFS_CACHE_FINISHED;
  6442. exclude_super_stripes(root, cache);
  6443. add_new_free_space(cache, root->fs_info, chunk_offset,
  6444. chunk_offset + size);
  6445. free_excluded_extents(root, cache);
  6446. ret = update_space_info(root->fs_info, cache->flags, size, bytes_used,
  6447. &cache->space_info);
  6448. BUG_ON(ret);
  6449. spin_lock(&cache->space_info->lock);
  6450. cache->space_info->bytes_readonly += cache->bytes_super;
  6451. spin_unlock(&cache->space_info->lock);
  6452. __link_block_group(cache->space_info, cache);
  6453. ret = btrfs_add_block_group_cache(root->fs_info, cache);
  6454. BUG_ON(ret);
  6455. ret = btrfs_insert_item(trans, extent_root, &cache->key, &cache->item,
  6456. sizeof(cache->item));
  6457. BUG_ON(ret);
  6458. set_avail_alloc_bits(extent_root->fs_info, type);
  6459. return 0;
  6460. }
  6461. int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
  6462. struct btrfs_root *root, u64 group_start)
  6463. {
  6464. struct btrfs_path *path;
  6465. struct btrfs_block_group_cache *block_group;
  6466. struct btrfs_free_cluster *cluster;
  6467. struct btrfs_root *tree_root = root->fs_info->tree_root;
  6468. struct btrfs_key key;
  6469. struct inode *inode;
  6470. int ret;
  6471. int factor;
  6472. root = root->fs_info->extent_root;
  6473. block_group = btrfs_lookup_block_group(root->fs_info, group_start);
  6474. BUG_ON(!block_group);
  6475. BUG_ON(!block_group->ro);
  6476. /*
  6477. * Free the reserved super bytes from this block group before
  6478. * remove it.
  6479. */
  6480. free_excluded_extents(root, block_group);
  6481. memcpy(&key, &block_group->key, sizeof(key));
  6482. if (block_group->flags & (BTRFS_BLOCK_GROUP_DUP |
  6483. BTRFS_BLOCK_GROUP_RAID1 |
  6484. BTRFS_BLOCK_GROUP_RAID10))
  6485. factor = 2;
  6486. else
  6487. factor = 1;
  6488. /* make sure this block group isn't part of an allocation cluster */
  6489. cluster = &root->fs_info->data_alloc_cluster;
  6490. spin_lock(&cluster->refill_lock);
  6491. btrfs_return_cluster_to_free_space(block_group, cluster);
  6492. spin_unlock(&cluster->refill_lock);
  6493. /*
  6494. * make sure this block group isn't part of a metadata
  6495. * allocation cluster
  6496. */
  6497. cluster = &root->fs_info->meta_alloc_cluster;
  6498. spin_lock(&cluster->refill_lock);
  6499. btrfs_return_cluster_to_free_space(block_group, cluster);
  6500. spin_unlock(&cluster->refill_lock);
  6501. path = btrfs_alloc_path();
  6502. if (!path) {
  6503. ret = -ENOMEM;
  6504. goto out;
  6505. }
  6506. inode = lookup_free_space_inode(tree_root, block_group, path);
  6507. if (!IS_ERR(inode)) {
  6508. ret = btrfs_orphan_add(trans, inode);
  6509. BUG_ON(ret);
  6510. clear_nlink(inode);
  6511. /* One for the block groups ref */
  6512. spin_lock(&block_group->lock);
  6513. if (block_group->iref) {
  6514. block_group->iref = 0;
  6515. block_group->inode = NULL;
  6516. spin_unlock(&block_group->lock);
  6517. iput(inode);
  6518. } else {
  6519. spin_unlock(&block_group->lock);
  6520. }
  6521. /* One for our lookup ref */
  6522. btrfs_add_delayed_iput(inode);
  6523. }
  6524. key.objectid = BTRFS_FREE_SPACE_OBJECTID;
  6525. key.offset = block_group->key.objectid;
  6526. key.type = 0;
  6527. ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1);
  6528. if (ret < 0)
  6529. goto out;
  6530. if (ret > 0)
  6531. btrfs_release_path(path);
  6532. if (ret == 0) {
  6533. ret = btrfs_del_item(trans, tree_root, path);
  6534. if (ret)
  6535. goto out;
  6536. btrfs_release_path(path);
  6537. }
  6538. spin_lock(&root->fs_info->block_group_cache_lock);
  6539. rb_erase(&block_group->cache_node,
  6540. &root->fs_info->block_group_cache_tree);
  6541. spin_unlock(&root->fs_info->block_group_cache_lock);
  6542. down_write(&block_group->space_info->groups_sem);
  6543. /*
  6544. * we must use list_del_init so people can check to see if they
  6545. * are still on the list after taking the semaphore
  6546. */
  6547. list_del_init(&block_group->list);
  6548. up_write(&block_group->space_info->groups_sem);
  6549. if (block_group->cached == BTRFS_CACHE_STARTED)
  6550. wait_block_group_cache_done(block_group);
  6551. btrfs_remove_free_space_cache(block_group);
  6552. spin_lock(&block_group->space_info->lock);
  6553. block_group->space_info->total_bytes -= block_group->key.offset;
  6554. block_group->space_info->bytes_readonly -= block_group->key.offset;
  6555. block_group->space_info->disk_total -= block_group->key.offset * factor;
  6556. spin_unlock(&block_group->space_info->lock);
  6557. memcpy(&key, &block_group->key, sizeof(key));
  6558. btrfs_clear_space_info_full(root->fs_info);
  6559. btrfs_put_block_group(block_group);
  6560. btrfs_put_block_group(block_group);
  6561. ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
  6562. if (ret > 0)
  6563. ret = -EIO;
  6564. if (ret < 0)
  6565. goto out;
  6566. ret = btrfs_del_item(trans, root, path);
  6567. out:
  6568. btrfs_free_path(path);
  6569. return ret;
  6570. }
  6571. int btrfs_init_space_info(struct btrfs_fs_info *fs_info)
  6572. {
  6573. struct btrfs_space_info *space_info;
  6574. struct btrfs_super_block *disk_super;
  6575. u64 features;
  6576. u64 flags;
  6577. int mixed = 0;
  6578. int ret;
  6579. disk_super = fs_info->super_copy;
  6580. if (!btrfs_super_root(disk_super))
  6581. return 1;
  6582. features = btrfs_super_incompat_flags(disk_super);
  6583. if (features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS)
  6584. mixed = 1;
  6585. flags = BTRFS_BLOCK_GROUP_SYSTEM;
  6586. ret = update_space_info(fs_info, flags, 0, 0, &space_info);
  6587. if (ret)
  6588. goto out;
  6589. if (mixed) {
  6590. flags = BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA;
  6591. ret = update_space_info(fs_info, flags, 0, 0, &space_info);
  6592. } else {
  6593. flags = BTRFS_BLOCK_GROUP_METADATA;
  6594. ret = update_space_info(fs_info, flags, 0, 0, &space_info);
  6595. if (ret)
  6596. goto out;
  6597. flags = BTRFS_BLOCK_GROUP_DATA;
  6598. ret = update_space_info(fs_info, flags, 0, 0, &space_info);
  6599. }
  6600. out:
  6601. return ret;
  6602. }
  6603. int btrfs_error_unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
  6604. {
  6605. return unpin_extent_range(root, start, end);
  6606. }
  6607. int btrfs_error_discard_extent(struct btrfs_root *root, u64 bytenr,
  6608. u64 num_bytes, u64 *actual_bytes)
  6609. {
  6610. return btrfs_discard_extent(root, bytenr, num_bytes, actual_bytes);
  6611. }
  6612. int btrfs_trim_fs(struct btrfs_root *root, struct fstrim_range *range)
  6613. {
  6614. struct btrfs_fs_info *fs_info = root->fs_info;
  6615. struct btrfs_block_group_cache *cache = NULL;
  6616. u64 group_trimmed;
  6617. u64 start;
  6618. u64 end;
  6619. u64 trimmed = 0;
  6620. int ret = 0;
  6621. cache = btrfs_lookup_block_group(fs_info, range->start);
  6622. while (cache) {
  6623. if (cache->key.objectid >= (range->start + range->len)) {
  6624. btrfs_put_block_group(cache);
  6625. break;
  6626. }
  6627. start = max(range->start, cache->key.objectid);
  6628. end = min(range->start + range->len,
  6629. cache->key.objectid + cache->key.offset);
  6630. if (end - start >= range->minlen) {
  6631. if (!block_group_cache_done(cache)) {
  6632. ret = cache_block_group(cache, NULL, root, 0);
  6633. if (!ret)
  6634. wait_block_group_cache_done(cache);
  6635. }
  6636. ret = btrfs_trim_block_group(cache,
  6637. &group_trimmed,
  6638. start,
  6639. end,
  6640. range->minlen);
  6641. trimmed += group_trimmed;
  6642. if (ret) {
  6643. btrfs_put_block_group(cache);
  6644. break;
  6645. }
  6646. }
  6647. cache = next_block_group(fs_info->tree_root, cache);
  6648. }
  6649. range->len = trimmed;
  6650. return ret;
  6651. }