raid5.c 181 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468
  1. /*
  2. * raid5.c : Multiple Devices driver for Linux
  3. * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman
  4. * Copyright (C) 1999, 2000 Ingo Molnar
  5. * Copyright (C) 2002, 2003 H. Peter Anvin
  6. *
  7. * RAID-4/5/6 management functions.
  8. * Thanks to Penguin Computing for making the RAID-6 development possible
  9. * by donating a test server!
  10. *
  11. * This program is free software; you can redistribute it and/or modify
  12. * it under the terms of the GNU General Public License as published by
  13. * the Free Software Foundation; either version 2, or (at your option)
  14. * any later version.
  15. *
  16. * You should have received a copy of the GNU General Public License
  17. * (for example /usr/src/linux/COPYING); if not, write to the Free
  18. * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19. */
  20. /*
  21. * BITMAP UNPLUGGING:
  22. *
  23. * The sequencing for updating the bitmap reliably is a little
  24. * subtle (and I got it wrong the first time) so it deserves some
  25. * explanation.
  26. *
  27. * We group bitmap updates into batches. Each batch has a number.
  28. * We may write out several batches at once, but that isn't very important.
  29. * conf->seq_write is the number of the last batch successfully written.
  30. * conf->seq_flush is the number of the last batch that was closed to
  31. * new additions.
  32. * When we discover that we will need to write to any block in a stripe
  33. * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq
  34. * the number of the batch it will be in. This is seq_flush+1.
  35. * When we are ready to do a write, if that batch hasn't been written yet,
  36. * we plug the array and queue the stripe for later.
  37. * When an unplug happens, we increment bm_flush, thus closing the current
  38. * batch.
  39. * When we notice that bm_flush > bm_write, we write out all pending updates
  40. * to the bitmap, and advance bm_write to where bm_flush was.
  41. * This may occasionally write a bit out twice, but is sure never to
  42. * miss any bits.
  43. */
  44. #include <linux/blkdev.h>
  45. #include <linux/kthread.h>
  46. #include <linux/raid/pq.h>
  47. #include <linux/async_tx.h>
  48. #include <linux/module.h>
  49. #include <linux/async.h>
  50. #include <linux/seq_file.h>
  51. #include <linux/cpu.h>
  52. #include <linux/slab.h>
  53. #include <linux/ratelimit.h>
  54. #include <trace/events/block.h>
  55. #include "md.h"
  56. #include "raid5.h"
  57. #include "raid0.h"
  58. #include "bitmap.h"
  59. /*
  60. * Stripe cache
  61. */
  62. #define NR_STRIPES 256
  63. #define STRIPE_SIZE PAGE_SIZE
  64. #define STRIPE_SHIFT (PAGE_SHIFT - 9)
  65. #define STRIPE_SECTORS (STRIPE_SIZE>>9)
  66. #define IO_THRESHOLD 1
  67. #define BYPASS_THRESHOLD 1
  68. #define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head))
  69. #define HASH_MASK (NR_HASH - 1)
  70. static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect)
  71. {
  72. int hash = (sect >> STRIPE_SHIFT) & HASH_MASK;
  73. return &conf->stripe_hashtbl[hash];
  74. }
  75. /* bio's attached to a stripe+device for I/O are linked together in bi_sector
  76. * order without overlap. There may be several bio's per stripe+device, and
  77. * a bio could span several devices.
  78. * When walking this list for a particular stripe+device, we must never proceed
  79. * beyond a bio that extends past this device, as the next bio might no longer
  80. * be valid.
  81. * This function is used to determine the 'next' bio in the list, given the sector
  82. * of the current stripe+device
  83. */
  84. static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector)
  85. {
  86. int sectors = bio->bi_size >> 9;
  87. if (bio->bi_sector + sectors < sector + STRIPE_SECTORS)
  88. return bio->bi_next;
  89. else
  90. return NULL;
  91. }
  92. /*
  93. * We maintain a biased count of active stripes in the bottom 16 bits of
  94. * bi_phys_segments, and a count of processed stripes in the upper 16 bits
  95. */
  96. static inline int raid5_bi_processed_stripes(struct bio *bio)
  97. {
  98. atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
  99. return (atomic_read(segments) >> 16) & 0xffff;
  100. }
  101. static inline int raid5_dec_bi_active_stripes(struct bio *bio)
  102. {
  103. atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
  104. return atomic_sub_return(1, segments) & 0xffff;
  105. }
  106. static inline void raid5_inc_bi_active_stripes(struct bio *bio)
  107. {
  108. atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
  109. atomic_inc(segments);
  110. }
  111. static inline void raid5_set_bi_processed_stripes(struct bio *bio,
  112. unsigned int cnt)
  113. {
  114. atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
  115. int old, new;
  116. do {
  117. old = atomic_read(segments);
  118. new = (old & 0xffff) | (cnt << 16);
  119. } while (atomic_cmpxchg(segments, old, new) != old);
  120. }
  121. static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt)
  122. {
  123. atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
  124. atomic_set(segments, cnt);
  125. }
  126. /* Find first data disk in a raid6 stripe */
  127. static inline int raid6_d0(struct stripe_head *sh)
  128. {
  129. if (sh->ddf_layout)
  130. /* ddf always start from first device */
  131. return 0;
  132. /* md starts just after Q block */
  133. if (sh->qd_idx == sh->disks - 1)
  134. return 0;
  135. else
  136. return sh->qd_idx + 1;
  137. }
  138. static inline int raid6_next_disk(int disk, int raid_disks)
  139. {
  140. disk++;
  141. return (disk < raid_disks) ? disk : 0;
  142. }
  143. /* When walking through the disks in a raid5, starting at raid6_d0,
  144. * We need to map each disk to a 'slot', where the data disks are slot
  145. * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk
  146. * is raid_disks-1. This help does that mapping.
  147. */
  148. static int raid6_idx_to_slot(int idx, struct stripe_head *sh,
  149. int *count, int syndrome_disks)
  150. {
  151. int slot = *count;
  152. if (sh->ddf_layout)
  153. (*count)++;
  154. if (idx == sh->pd_idx)
  155. return syndrome_disks;
  156. if (idx == sh->qd_idx)
  157. return syndrome_disks + 1;
  158. if (!sh->ddf_layout)
  159. (*count)++;
  160. return slot;
  161. }
  162. static void return_io(struct bio *return_bi)
  163. {
  164. struct bio *bi = return_bi;
  165. while (bi) {
  166. return_bi = bi->bi_next;
  167. bi->bi_next = NULL;
  168. bi->bi_size = 0;
  169. bio_endio(bi, 0);
  170. bi = return_bi;
  171. }
  172. }
  173. static void print_raid5_conf (struct r5conf *conf);
  174. static int stripe_operations_active(struct stripe_head *sh)
  175. {
  176. return sh->check_state || sh->reconstruct_state ||
  177. test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||
  178. test_bit(STRIPE_COMPUTE_RUN, &sh->state);
  179. }
  180. static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh)
  181. {
  182. BUG_ON(!list_empty(&sh->lru));
  183. BUG_ON(atomic_read(&conf->active_stripes)==0);
  184. if (test_bit(STRIPE_HANDLE, &sh->state)) {
  185. if (test_bit(STRIPE_DELAYED, &sh->state) &&
  186. !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  187. list_add_tail(&sh->lru, &conf->delayed_list);
  188. else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
  189. sh->bm_seq - conf->seq_write > 0)
  190. list_add_tail(&sh->lru, &conf->bitmap_list);
  191. else {
  192. clear_bit(STRIPE_DELAYED, &sh->state);
  193. clear_bit(STRIPE_BIT_DELAY, &sh->state);
  194. list_add_tail(&sh->lru, &conf->handle_list);
  195. }
  196. md_wakeup_thread(conf->mddev->thread);
  197. } else {
  198. BUG_ON(stripe_operations_active(sh));
  199. if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  200. if (atomic_dec_return(&conf->preread_active_stripes)
  201. < IO_THRESHOLD)
  202. md_wakeup_thread(conf->mddev->thread);
  203. atomic_dec(&conf->active_stripes);
  204. if (!test_bit(STRIPE_EXPANDING, &sh->state)) {
  205. list_add_tail(&sh->lru, &conf->inactive_list);
  206. wake_up(&conf->wait_for_stripe);
  207. if (conf->retry_read_aligned)
  208. md_wakeup_thread(conf->mddev->thread);
  209. }
  210. }
  211. }
  212. static void __release_stripe(struct r5conf *conf, struct stripe_head *sh)
  213. {
  214. if (atomic_dec_and_test(&sh->count))
  215. do_release_stripe(conf, sh);
  216. }
  217. static void release_stripe(struct stripe_head *sh)
  218. {
  219. struct r5conf *conf = sh->raid_conf;
  220. unsigned long flags;
  221. local_irq_save(flags);
  222. if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) {
  223. do_release_stripe(conf, sh);
  224. spin_unlock(&conf->device_lock);
  225. }
  226. local_irq_restore(flags);
  227. }
  228. static inline void remove_hash(struct stripe_head *sh)
  229. {
  230. pr_debug("remove_hash(), stripe %llu\n",
  231. (unsigned long long)sh->sector);
  232. hlist_del_init(&sh->hash);
  233. }
  234. static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh)
  235. {
  236. struct hlist_head *hp = stripe_hash(conf, sh->sector);
  237. pr_debug("insert_hash(), stripe %llu\n",
  238. (unsigned long long)sh->sector);
  239. hlist_add_head(&sh->hash, hp);
  240. }
  241. /* find an idle stripe, make sure it is unhashed, and return it. */
  242. static struct stripe_head *get_free_stripe(struct r5conf *conf)
  243. {
  244. struct stripe_head *sh = NULL;
  245. struct list_head *first;
  246. if (list_empty(&conf->inactive_list))
  247. goto out;
  248. first = conf->inactive_list.next;
  249. sh = list_entry(first, struct stripe_head, lru);
  250. list_del_init(first);
  251. remove_hash(sh);
  252. atomic_inc(&conf->active_stripes);
  253. out:
  254. return sh;
  255. }
  256. static void shrink_buffers(struct stripe_head *sh)
  257. {
  258. struct page *p;
  259. int i;
  260. int num = sh->raid_conf->pool_size;
  261. for (i = 0; i < num ; i++) {
  262. p = sh->dev[i].page;
  263. if (!p)
  264. continue;
  265. sh->dev[i].page = NULL;
  266. put_page(p);
  267. }
  268. }
  269. static int grow_buffers(struct stripe_head *sh)
  270. {
  271. int i;
  272. int num = sh->raid_conf->pool_size;
  273. for (i = 0; i < num; i++) {
  274. struct page *page;
  275. if (!(page = alloc_page(GFP_KERNEL))) {
  276. return 1;
  277. }
  278. sh->dev[i].page = page;
  279. }
  280. return 0;
  281. }
  282. static void raid5_build_block(struct stripe_head *sh, int i, int previous);
  283. static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
  284. struct stripe_head *sh);
  285. static void init_stripe(struct stripe_head *sh, sector_t sector, int previous)
  286. {
  287. struct r5conf *conf = sh->raid_conf;
  288. int i;
  289. BUG_ON(atomic_read(&sh->count) != 0);
  290. BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
  291. BUG_ON(stripe_operations_active(sh));
  292. pr_debug("init_stripe called, stripe %llu\n",
  293. (unsigned long long)sh->sector);
  294. remove_hash(sh);
  295. sh->generation = conf->generation - previous;
  296. sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;
  297. sh->sector = sector;
  298. stripe_set_idx(sector, conf, previous, sh);
  299. sh->state = 0;
  300. for (i = sh->disks; i--; ) {
  301. struct r5dev *dev = &sh->dev[i];
  302. if (dev->toread || dev->read || dev->towrite || dev->written ||
  303. test_bit(R5_LOCKED, &dev->flags)) {
  304. printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n",
  305. (unsigned long long)sh->sector, i, dev->toread,
  306. dev->read, dev->towrite, dev->written,
  307. test_bit(R5_LOCKED, &dev->flags));
  308. WARN_ON(1);
  309. }
  310. dev->flags = 0;
  311. raid5_build_block(sh, i, previous);
  312. }
  313. insert_hash(conf, sh);
  314. }
  315. static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector,
  316. short generation)
  317. {
  318. struct stripe_head *sh;
  319. pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector);
  320. hlist_for_each_entry(sh, stripe_hash(conf, sector), hash)
  321. if (sh->sector == sector && sh->generation == generation)
  322. return sh;
  323. pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector);
  324. return NULL;
  325. }
  326. /*
  327. * Need to check if array has failed when deciding whether to:
  328. * - start an array
  329. * - remove non-faulty devices
  330. * - add a spare
  331. * - allow a reshape
  332. * This determination is simple when no reshape is happening.
  333. * However if there is a reshape, we need to carefully check
  334. * both the before and after sections.
  335. * This is because some failed devices may only affect one
  336. * of the two sections, and some non-in_sync devices may
  337. * be insync in the section most affected by failed devices.
  338. */
  339. static int calc_degraded(struct r5conf *conf)
  340. {
  341. int degraded, degraded2;
  342. int i;
  343. rcu_read_lock();
  344. degraded = 0;
  345. for (i = 0; i < conf->previous_raid_disks; i++) {
  346. struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
  347. if (rdev && test_bit(Faulty, &rdev->flags))
  348. rdev = rcu_dereference(conf->disks[i].replacement);
  349. if (!rdev || test_bit(Faulty, &rdev->flags))
  350. degraded++;
  351. else if (test_bit(In_sync, &rdev->flags))
  352. ;
  353. else
  354. /* not in-sync or faulty.
  355. * If the reshape increases the number of devices,
  356. * this is being recovered by the reshape, so
  357. * this 'previous' section is not in_sync.
  358. * If the number of devices is being reduced however,
  359. * the device can only be part of the array if
  360. * we are reverting a reshape, so this section will
  361. * be in-sync.
  362. */
  363. if (conf->raid_disks >= conf->previous_raid_disks)
  364. degraded++;
  365. }
  366. rcu_read_unlock();
  367. if (conf->raid_disks == conf->previous_raid_disks)
  368. return degraded;
  369. rcu_read_lock();
  370. degraded2 = 0;
  371. for (i = 0; i < conf->raid_disks; i++) {
  372. struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
  373. if (rdev && test_bit(Faulty, &rdev->flags))
  374. rdev = rcu_dereference(conf->disks[i].replacement);
  375. if (!rdev || test_bit(Faulty, &rdev->flags))
  376. degraded2++;
  377. else if (test_bit(In_sync, &rdev->flags))
  378. ;
  379. else
  380. /* not in-sync or faulty.
  381. * If reshape increases the number of devices, this
  382. * section has already been recovered, else it
  383. * almost certainly hasn't.
  384. */
  385. if (conf->raid_disks <= conf->previous_raid_disks)
  386. degraded2++;
  387. }
  388. rcu_read_unlock();
  389. if (degraded2 > degraded)
  390. return degraded2;
  391. return degraded;
  392. }
  393. static int has_failed(struct r5conf *conf)
  394. {
  395. int degraded;
  396. if (conf->mddev->reshape_position == MaxSector)
  397. return conf->mddev->degraded > conf->max_degraded;
  398. degraded = calc_degraded(conf);
  399. if (degraded > conf->max_degraded)
  400. return 1;
  401. return 0;
  402. }
  403. static struct stripe_head *
  404. get_active_stripe(struct r5conf *conf, sector_t sector,
  405. int previous, int noblock, int noquiesce)
  406. {
  407. struct stripe_head *sh;
  408. pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector);
  409. spin_lock_irq(&conf->device_lock);
  410. do {
  411. wait_event_lock_irq(conf->wait_for_stripe,
  412. conf->quiesce == 0 || noquiesce,
  413. conf->device_lock);
  414. sh = __find_stripe(conf, sector, conf->generation - previous);
  415. if (!sh) {
  416. if (!conf->inactive_blocked)
  417. sh = get_free_stripe(conf);
  418. if (noblock && sh == NULL)
  419. break;
  420. if (!sh) {
  421. conf->inactive_blocked = 1;
  422. wait_event_lock_irq(conf->wait_for_stripe,
  423. !list_empty(&conf->inactive_list) &&
  424. (atomic_read(&conf->active_stripes)
  425. < (conf->max_nr_stripes *3/4)
  426. || !conf->inactive_blocked),
  427. conf->device_lock);
  428. conf->inactive_blocked = 0;
  429. } else
  430. init_stripe(sh, sector, previous);
  431. } else {
  432. if (atomic_read(&sh->count)) {
  433. BUG_ON(!list_empty(&sh->lru)
  434. && !test_bit(STRIPE_EXPANDING, &sh->state)
  435. && !test_bit(STRIPE_ON_UNPLUG_LIST, &sh->state));
  436. } else {
  437. if (!test_bit(STRIPE_HANDLE, &sh->state))
  438. atomic_inc(&conf->active_stripes);
  439. if (list_empty(&sh->lru) &&
  440. !test_bit(STRIPE_EXPANDING, &sh->state))
  441. BUG();
  442. list_del_init(&sh->lru);
  443. }
  444. }
  445. } while (sh == NULL);
  446. if (sh)
  447. atomic_inc(&sh->count);
  448. spin_unlock_irq(&conf->device_lock);
  449. return sh;
  450. }
  451. /* Determine if 'data_offset' or 'new_data_offset' should be used
  452. * in this stripe_head.
  453. */
  454. static int use_new_offset(struct r5conf *conf, struct stripe_head *sh)
  455. {
  456. sector_t progress = conf->reshape_progress;
  457. /* Need a memory barrier to make sure we see the value
  458. * of conf->generation, or ->data_offset that was set before
  459. * reshape_progress was updated.
  460. */
  461. smp_rmb();
  462. if (progress == MaxSector)
  463. return 0;
  464. if (sh->generation == conf->generation - 1)
  465. return 0;
  466. /* We are in a reshape, and this is a new-generation stripe,
  467. * so use new_data_offset.
  468. */
  469. return 1;
  470. }
  471. static void
  472. raid5_end_read_request(struct bio *bi, int error);
  473. static void
  474. raid5_end_write_request(struct bio *bi, int error);
  475. static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
  476. {
  477. struct r5conf *conf = sh->raid_conf;
  478. int i, disks = sh->disks;
  479. might_sleep();
  480. for (i = disks; i--; ) {
  481. int rw;
  482. int replace_only = 0;
  483. struct bio *bi, *rbi;
  484. struct md_rdev *rdev, *rrdev = NULL;
  485. if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
  486. if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags))
  487. rw = WRITE_FUA;
  488. else
  489. rw = WRITE;
  490. if (test_bit(R5_Discard, &sh->dev[i].flags))
  491. rw |= REQ_DISCARD;
  492. } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
  493. rw = READ;
  494. else if (test_and_clear_bit(R5_WantReplace,
  495. &sh->dev[i].flags)) {
  496. rw = WRITE;
  497. replace_only = 1;
  498. } else
  499. continue;
  500. if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags))
  501. rw |= REQ_SYNC;
  502. bi = &sh->dev[i].req;
  503. rbi = &sh->dev[i].rreq; /* For writing to replacement */
  504. bi->bi_rw = rw;
  505. rbi->bi_rw = rw;
  506. if (rw & WRITE) {
  507. bi->bi_end_io = raid5_end_write_request;
  508. rbi->bi_end_io = raid5_end_write_request;
  509. } else
  510. bi->bi_end_io = raid5_end_read_request;
  511. rcu_read_lock();
  512. rrdev = rcu_dereference(conf->disks[i].replacement);
  513. smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */
  514. rdev = rcu_dereference(conf->disks[i].rdev);
  515. if (!rdev) {
  516. rdev = rrdev;
  517. rrdev = NULL;
  518. }
  519. if (rw & WRITE) {
  520. if (replace_only)
  521. rdev = NULL;
  522. if (rdev == rrdev)
  523. /* We raced and saw duplicates */
  524. rrdev = NULL;
  525. } else {
  526. if (test_bit(R5_ReadRepl, &sh->dev[i].flags) && rrdev)
  527. rdev = rrdev;
  528. rrdev = NULL;
  529. }
  530. if (rdev && test_bit(Faulty, &rdev->flags))
  531. rdev = NULL;
  532. if (rdev)
  533. atomic_inc(&rdev->nr_pending);
  534. if (rrdev && test_bit(Faulty, &rrdev->flags))
  535. rrdev = NULL;
  536. if (rrdev)
  537. atomic_inc(&rrdev->nr_pending);
  538. rcu_read_unlock();
  539. /* We have already checked bad blocks for reads. Now
  540. * need to check for writes. We never accept write errors
  541. * on the replacement, so we don't to check rrdev.
  542. */
  543. while ((rw & WRITE) && rdev &&
  544. test_bit(WriteErrorSeen, &rdev->flags)) {
  545. sector_t first_bad;
  546. int bad_sectors;
  547. int bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
  548. &first_bad, &bad_sectors);
  549. if (!bad)
  550. break;
  551. if (bad < 0) {
  552. set_bit(BlockedBadBlocks, &rdev->flags);
  553. if (!conf->mddev->external &&
  554. conf->mddev->flags) {
  555. /* It is very unlikely, but we might
  556. * still need to write out the
  557. * bad block log - better give it
  558. * a chance*/
  559. md_check_recovery(conf->mddev);
  560. }
  561. /*
  562. * Because md_wait_for_blocked_rdev
  563. * will dec nr_pending, we must
  564. * increment it first.
  565. */
  566. atomic_inc(&rdev->nr_pending);
  567. md_wait_for_blocked_rdev(rdev, conf->mddev);
  568. } else {
  569. /* Acknowledged bad block - skip the write */
  570. rdev_dec_pending(rdev, conf->mddev);
  571. rdev = NULL;
  572. }
  573. }
  574. if (rdev) {
  575. if (s->syncing || s->expanding || s->expanded
  576. || s->replacing)
  577. md_sync_acct(rdev->bdev, STRIPE_SECTORS);
  578. set_bit(STRIPE_IO_STARTED, &sh->state);
  579. bi->bi_bdev = rdev->bdev;
  580. pr_debug("%s: for %llu schedule op %ld on disc %d\n",
  581. __func__, (unsigned long long)sh->sector,
  582. bi->bi_rw, i);
  583. atomic_inc(&sh->count);
  584. if (use_new_offset(conf, sh))
  585. bi->bi_sector = (sh->sector
  586. + rdev->new_data_offset);
  587. else
  588. bi->bi_sector = (sh->sector
  589. + rdev->data_offset);
  590. if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
  591. bi->bi_rw |= REQ_FLUSH;
  592. bi->bi_flags = 1 << BIO_UPTODATE;
  593. bi->bi_idx = 0;
  594. bi->bi_io_vec[0].bv_len = STRIPE_SIZE;
  595. bi->bi_io_vec[0].bv_offset = 0;
  596. bi->bi_size = STRIPE_SIZE;
  597. bi->bi_next = NULL;
  598. if (rrdev)
  599. set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags);
  600. trace_block_bio_remap(bdev_get_queue(bi->bi_bdev),
  601. bi, disk_devt(conf->mddev->gendisk),
  602. sh->dev[i].sector);
  603. generic_make_request(bi);
  604. }
  605. if (rrdev) {
  606. if (s->syncing || s->expanding || s->expanded
  607. || s->replacing)
  608. md_sync_acct(rrdev->bdev, STRIPE_SECTORS);
  609. set_bit(STRIPE_IO_STARTED, &sh->state);
  610. rbi->bi_bdev = rrdev->bdev;
  611. pr_debug("%s: for %llu schedule op %ld on "
  612. "replacement disc %d\n",
  613. __func__, (unsigned long long)sh->sector,
  614. rbi->bi_rw, i);
  615. atomic_inc(&sh->count);
  616. if (use_new_offset(conf, sh))
  617. rbi->bi_sector = (sh->sector
  618. + rrdev->new_data_offset);
  619. else
  620. rbi->bi_sector = (sh->sector
  621. + rrdev->data_offset);
  622. rbi->bi_flags = 1 << BIO_UPTODATE;
  623. rbi->bi_idx = 0;
  624. rbi->bi_io_vec[0].bv_len = STRIPE_SIZE;
  625. rbi->bi_io_vec[0].bv_offset = 0;
  626. rbi->bi_size = STRIPE_SIZE;
  627. rbi->bi_next = NULL;
  628. trace_block_bio_remap(bdev_get_queue(rbi->bi_bdev),
  629. rbi, disk_devt(conf->mddev->gendisk),
  630. sh->dev[i].sector);
  631. generic_make_request(rbi);
  632. }
  633. if (!rdev && !rrdev) {
  634. if (rw & WRITE)
  635. set_bit(STRIPE_DEGRADED, &sh->state);
  636. pr_debug("skip op %ld on disc %d for sector %llu\n",
  637. bi->bi_rw, i, (unsigned long long)sh->sector);
  638. clear_bit(R5_LOCKED, &sh->dev[i].flags);
  639. set_bit(STRIPE_HANDLE, &sh->state);
  640. }
  641. }
  642. }
  643. static struct dma_async_tx_descriptor *
  644. async_copy_data(int frombio, struct bio *bio, struct page *page,
  645. sector_t sector, struct dma_async_tx_descriptor *tx)
  646. {
  647. struct bio_vec *bvl;
  648. struct page *bio_page;
  649. int i;
  650. int page_offset;
  651. struct async_submit_ctl submit;
  652. enum async_tx_flags flags = 0;
  653. if (bio->bi_sector >= sector)
  654. page_offset = (signed)(bio->bi_sector - sector) * 512;
  655. else
  656. page_offset = (signed)(sector - bio->bi_sector) * -512;
  657. if (frombio)
  658. flags |= ASYNC_TX_FENCE;
  659. init_async_submit(&submit, flags, tx, NULL, NULL, NULL);
  660. bio_for_each_segment(bvl, bio, i) {
  661. int len = bvl->bv_len;
  662. int clen;
  663. int b_offset = 0;
  664. if (page_offset < 0) {
  665. b_offset = -page_offset;
  666. page_offset += b_offset;
  667. len -= b_offset;
  668. }
  669. if (len > 0 && page_offset + len > STRIPE_SIZE)
  670. clen = STRIPE_SIZE - page_offset;
  671. else
  672. clen = len;
  673. if (clen > 0) {
  674. b_offset += bvl->bv_offset;
  675. bio_page = bvl->bv_page;
  676. if (frombio)
  677. tx = async_memcpy(page, bio_page, page_offset,
  678. b_offset, clen, &submit);
  679. else
  680. tx = async_memcpy(bio_page, page, b_offset,
  681. page_offset, clen, &submit);
  682. }
  683. /* chain the operations */
  684. submit.depend_tx = tx;
  685. if (clen < len) /* hit end of page */
  686. break;
  687. page_offset += len;
  688. }
  689. return tx;
  690. }
  691. static void ops_complete_biofill(void *stripe_head_ref)
  692. {
  693. struct stripe_head *sh = stripe_head_ref;
  694. struct bio *return_bi = NULL;
  695. int i;
  696. pr_debug("%s: stripe %llu\n", __func__,
  697. (unsigned long long)sh->sector);
  698. /* clear completed biofills */
  699. for (i = sh->disks; i--; ) {
  700. struct r5dev *dev = &sh->dev[i];
  701. /* acknowledge completion of a biofill operation */
  702. /* and check if we need to reply to a read request,
  703. * new R5_Wantfill requests are held off until
  704. * !STRIPE_BIOFILL_RUN
  705. */
  706. if (test_and_clear_bit(R5_Wantfill, &dev->flags)) {
  707. struct bio *rbi, *rbi2;
  708. BUG_ON(!dev->read);
  709. rbi = dev->read;
  710. dev->read = NULL;
  711. while (rbi && rbi->bi_sector <
  712. dev->sector + STRIPE_SECTORS) {
  713. rbi2 = r5_next_bio(rbi, dev->sector);
  714. if (!raid5_dec_bi_active_stripes(rbi)) {
  715. rbi->bi_next = return_bi;
  716. return_bi = rbi;
  717. }
  718. rbi = rbi2;
  719. }
  720. }
  721. }
  722. clear_bit(STRIPE_BIOFILL_RUN, &sh->state);
  723. return_io(return_bi);
  724. set_bit(STRIPE_HANDLE, &sh->state);
  725. release_stripe(sh);
  726. }
  727. static void ops_run_biofill(struct stripe_head *sh)
  728. {
  729. struct dma_async_tx_descriptor *tx = NULL;
  730. struct async_submit_ctl submit;
  731. int i;
  732. pr_debug("%s: stripe %llu\n", __func__,
  733. (unsigned long long)sh->sector);
  734. for (i = sh->disks; i--; ) {
  735. struct r5dev *dev = &sh->dev[i];
  736. if (test_bit(R5_Wantfill, &dev->flags)) {
  737. struct bio *rbi;
  738. spin_lock_irq(&sh->stripe_lock);
  739. dev->read = rbi = dev->toread;
  740. dev->toread = NULL;
  741. spin_unlock_irq(&sh->stripe_lock);
  742. while (rbi && rbi->bi_sector <
  743. dev->sector + STRIPE_SECTORS) {
  744. tx = async_copy_data(0, rbi, dev->page,
  745. dev->sector, tx);
  746. rbi = r5_next_bio(rbi, dev->sector);
  747. }
  748. }
  749. }
  750. atomic_inc(&sh->count);
  751. init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);
  752. async_trigger_callback(&submit);
  753. }
  754. static void mark_target_uptodate(struct stripe_head *sh, int target)
  755. {
  756. struct r5dev *tgt;
  757. if (target < 0)
  758. return;
  759. tgt = &sh->dev[target];
  760. set_bit(R5_UPTODATE, &tgt->flags);
  761. BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
  762. clear_bit(R5_Wantcompute, &tgt->flags);
  763. }
  764. static void ops_complete_compute(void *stripe_head_ref)
  765. {
  766. struct stripe_head *sh = stripe_head_ref;
  767. pr_debug("%s: stripe %llu\n", __func__,
  768. (unsigned long long)sh->sector);
  769. /* mark the computed target(s) as uptodate */
  770. mark_target_uptodate(sh, sh->ops.target);
  771. mark_target_uptodate(sh, sh->ops.target2);
  772. clear_bit(STRIPE_COMPUTE_RUN, &sh->state);
  773. if (sh->check_state == check_state_compute_run)
  774. sh->check_state = check_state_compute_result;
  775. set_bit(STRIPE_HANDLE, &sh->state);
  776. release_stripe(sh);
  777. }
  778. /* return a pointer to the address conversion region of the scribble buffer */
  779. static addr_conv_t *to_addr_conv(struct stripe_head *sh,
  780. struct raid5_percpu *percpu)
  781. {
  782. return percpu->scribble + sizeof(struct page *) * (sh->disks + 2);
  783. }
  784. static struct dma_async_tx_descriptor *
  785. ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
  786. {
  787. int disks = sh->disks;
  788. struct page **xor_srcs = percpu->scribble;
  789. int target = sh->ops.target;
  790. struct r5dev *tgt = &sh->dev[target];
  791. struct page *xor_dest = tgt->page;
  792. int count = 0;
  793. struct dma_async_tx_descriptor *tx;
  794. struct async_submit_ctl submit;
  795. int i;
  796. pr_debug("%s: stripe %llu block: %d\n",
  797. __func__, (unsigned long long)sh->sector, target);
  798. BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
  799. for (i = disks; i--; )
  800. if (i != target)
  801. xor_srcs[count++] = sh->dev[i].page;
  802. atomic_inc(&sh->count);
  803. init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL,
  804. ops_complete_compute, sh, to_addr_conv(sh, percpu));
  805. if (unlikely(count == 1))
  806. tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
  807. else
  808. tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
  809. return tx;
  810. }
  811. /* set_syndrome_sources - populate source buffers for gen_syndrome
  812. * @srcs - (struct page *) array of size sh->disks
  813. * @sh - stripe_head to parse
  814. *
  815. * Populates srcs in proper layout order for the stripe and returns the
  816. * 'count' of sources to be used in a call to async_gen_syndrome. The P
  817. * destination buffer is recorded in srcs[count] and the Q destination
  818. * is recorded in srcs[count+1]].
  819. */
  820. static int set_syndrome_sources(struct page **srcs, struct stripe_head *sh)
  821. {
  822. int disks = sh->disks;
  823. int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);
  824. int d0_idx = raid6_d0(sh);
  825. int count;
  826. int i;
  827. for (i = 0; i < disks; i++)
  828. srcs[i] = NULL;
  829. count = 0;
  830. i = d0_idx;
  831. do {
  832. int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
  833. srcs[slot] = sh->dev[i].page;
  834. i = raid6_next_disk(i, disks);
  835. } while (i != d0_idx);
  836. return syndrome_disks;
  837. }
  838. static struct dma_async_tx_descriptor *
  839. ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
  840. {
  841. int disks = sh->disks;
  842. struct page **blocks = percpu->scribble;
  843. int target;
  844. int qd_idx = sh->qd_idx;
  845. struct dma_async_tx_descriptor *tx;
  846. struct async_submit_ctl submit;
  847. struct r5dev *tgt;
  848. struct page *dest;
  849. int i;
  850. int count;
  851. if (sh->ops.target < 0)
  852. target = sh->ops.target2;
  853. else if (sh->ops.target2 < 0)
  854. target = sh->ops.target;
  855. else
  856. /* we should only have one valid target */
  857. BUG();
  858. BUG_ON(target < 0);
  859. pr_debug("%s: stripe %llu block: %d\n",
  860. __func__, (unsigned long long)sh->sector, target);
  861. tgt = &sh->dev[target];
  862. BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
  863. dest = tgt->page;
  864. atomic_inc(&sh->count);
  865. if (target == qd_idx) {
  866. count = set_syndrome_sources(blocks, sh);
  867. blocks[count] = NULL; /* regenerating p is not necessary */
  868. BUG_ON(blocks[count+1] != dest); /* q should already be set */
  869. init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
  870. ops_complete_compute, sh,
  871. to_addr_conv(sh, percpu));
  872. tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
  873. } else {
  874. /* Compute any data- or p-drive using XOR */
  875. count = 0;
  876. for (i = disks; i-- ; ) {
  877. if (i == target || i == qd_idx)
  878. continue;
  879. blocks[count++] = sh->dev[i].page;
  880. }
  881. init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
  882. NULL, ops_complete_compute, sh,
  883. to_addr_conv(sh, percpu));
  884. tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit);
  885. }
  886. return tx;
  887. }
  888. static struct dma_async_tx_descriptor *
  889. ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
  890. {
  891. int i, count, disks = sh->disks;
  892. int syndrome_disks = sh->ddf_layout ? disks : disks-2;
  893. int d0_idx = raid6_d0(sh);
  894. int faila = -1, failb = -1;
  895. int target = sh->ops.target;
  896. int target2 = sh->ops.target2;
  897. struct r5dev *tgt = &sh->dev[target];
  898. struct r5dev *tgt2 = &sh->dev[target2];
  899. struct dma_async_tx_descriptor *tx;
  900. struct page **blocks = percpu->scribble;
  901. struct async_submit_ctl submit;
  902. pr_debug("%s: stripe %llu block1: %d block2: %d\n",
  903. __func__, (unsigned long long)sh->sector, target, target2);
  904. BUG_ON(target < 0 || target2 < 0);
  905. BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
  906. BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags));
  907. /* we need to open-code set_syndrome_sources to handle the
  908. * slot number conversion for 'faila' and 'failb'
  909. */
  910. for (i = 0; i < disks ; i++)
  911. blocks[i] = NULL;
  912. count = 0;
  913. i = d0_idx;
  914. do {
  915. int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
  916. blocks[slot] = sh->dev[i].page;
  917. if (i == target)
  918. faila = slot;
  919. if (i == target2)
  920. failb = slot;
  921. i = raid6_next_disk(i, disks);
  922. } while (i != d0_idx);
  923. BUG_ON(faila == failb);
  924. if (failb < faila)
  925. swap(faila, failb);
  926. pr_debug("%s: stripe: %llu faila: %d failb: %d\n",
  927. __func__, (unsigned long long)sh->sector, faila, failb);
  928. atomic_inc(&sh->count);
  929. if (failb == syndrome_disks+1) {
  930. /* Q disk is one of the missing disks */
  931. if (faila == syndrome_disks) {
  932. /* Missing P+Q, just recompute */
  933. init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
  934. ops_complete_compute, sh,
  935. to_addr_conv(sh, percpu));
  936. return async_gen_syndrome(blocks, 0, syndrome_disks+2,
  937. STRIPE_SIZE, &submit);
  938. } else {
  939. struct page *dest;
  940. int data_target;
  941. int qd_idx = sh->qd_idx;
  942. /* Missing D+Q: recompute D from P, then recompute Q */
  943. if (target == qd_idx)
  944. data_target = target2;
  945. else
  946. data_target = target;
  947. count = 0;
  948. for (i = disks; i-- ; ) {
  949. if (i == data_target || i == qd_idx)
  950. continue;
  951. blocks[count++] = sh->dev[i].page;
  952. }
  953. dest = sh->dev[data_target].page;
  954. init_async_submit(&submit,
  955. ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
  956. NULL, NULL, NULL,
  957. to_addr_conv(sh, percpu));
  958. tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE,
  959. &submit);
  960. count = set_syndrome_sources(blocks, sh);
  961. init_async_submit(&submit, ASYNC_TX_FENCE, tx,
  962. ops_complete_compute, sh,
  963. to_addr_conv(sh, percpu));
  964. return async_gen_syndrome(blocks, 0, count+2,
  965. STRIPE_SIZE, &submit);
  966. }
  967. } else {
  968. init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
  969. ops_complete_compute, sh,
  970. to_addr_conv(sh, percpu));
  971. if (failb == syndrome_disks) {
  972. /* We're missing D+P. */
  973. return async_raid6_datap_recov(syndrome_disks+2,
  974. STRIPE_SIZE, faila,
  975. blocks, &submit);
  976. } else {
  977. /* We're missing D+D. */
  978. return async_raid6_2data_recov(syndrome_disks+2,
  979. STRIPE_SIZE, faila, failb,
  980. blocks, &submit);
  981. }
  982. }
  983. }
  984. static void ops_complete_prexor(void *stripe_head_ref)
  985. {
  986. struct stripe_head *sh = stripe_head_ref;
  987. pr_debug("%s: stripe %llu\n", __func__,
  988. (unsigned long long)sh->sector);
  989. }
  990. static struct dma_async_tx_descriptor *
  991. ops_run_prexor(struct stripe_head *sh, struct raid5_percpu *percpu,
  992. struct dma_async_tx_descriptor *tx)
  993. {
  994. int disks = sh->disks;
  995. struct page **xor_srcs = percpu->scribble;
  996. int count = 0, pd_idx = sh->pd_idx, i;
  997. struct async_submit_ctl submit;
  998. /* existing parity data subtracted */
  999. struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
  1000. pr_debug("%s: stripe %llu\n", __func__,
  1001. (unsigned long long)sh->sector);
  1002. for (i = disks; i--; ) {
  1003. struct r5dev *dev = &sh->dev[i];
  1004. /* Only process blocks that are known to be uptodate */
  1005. if (test_bit(R5_Wantdrain, &dev->flags))
  1006. xor_srcs[count++] = dev->page;
  1007. }
  1008. init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
  1009. ops_complete_prexor, sh, to_addr_conv(sh, percpu));
  1010. tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
  1011. return tx;
  1012. }
  1013. static struct dma_async_tx_descriptor *
  1014. ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
  1015. {
  1016. int disks = sh->disks;
  1017. int i;
  1018. pr_debug("%s: stripe %llu\n", __func__,
  1019. (unsigned long long)sh->sector);
  1020. for (i = disks; i--; ) {
  1021. struct r5dev *dev = &sh->dev[i];
  1022. struct bio *chosen;
  1023. if (test_and_clear_bit(R5_Wantdrain, &dev->flags)) {
  1024. struct bio *wbi;
  1025. spin_lock_irq(&sh->stripe_lock);
  1026. chosen = dev->towrite;
  1027. dev->towrite = NULL;
  1028. BUG_ON(dev->written);
  1029. wbi = dev->written = chosen;
  1030. spin_unlock_irq(&sh->stripe_lock);
  1031. while (wbi && wbi->bi_sector <
  1032. dev->sector + STRIPE_SECTORS) {
  1033. if (wbi->bi_rw & REQ_FUA)
  1034. set_bit(R5_WantFUA, &dev->flags);
  1035. if (wbi->bi_rw & REQ_SYNC)
  1036. set_bit(R5_SyncIO, &dev->flags);
  1037. if (wbi->bi_rw & REQ_DISCARD)
  1038. set_bit(R5_Discard, &dev->flags);
  1039. else
  1040. tx = async_copy_data(1, wbi, dev->page,
  1041. dev->sector, tx);
  1042. wbi = r5_next_bio(wbi, dev->sector);
  1043. }
  1044. }
  1045. }
  1046. return tx;
  1047. }
  1048. static void ops_complete_reconstruct(void *stripe_head_ref)
  1049. {
  1050. struct stripe_head *sh = stripe_head_ref;
  1051. int disks = sh->disks;
  1052. int pd_idx = sh->pd_idx;
  1053. int qd_idx = sh->qd_idx;
  1054. int i;
  1055. bool fua = false, sync = false, discard = false;
  1056. pr_debug("%s: stripe %llu\n", __func__,
  1057. (unsigned long long)sh->sector);
  1058. for (i = disks; i--; ) {
  1059. fua |= test_bit(R5_WantFUA, &sh->dev[i].flags);
  1060. sync |= test_bit(R5_SyncIO, &sh->dev[i].flags);
  1061. discard |= test_bit(R5_Discard, &sh->dev[i].flags);
  1062. }
  1063. for (i = disks; i--; ) {
  1064. struct r5dev *dev = &sh->dev[i];
  1065. if (dev->written || i == pd_idx || i == qd_idx) {
  1066. if (!discard)
  1067. set_bit(R5_UPTODATE, &dev->flags);
  1068. if (fua)
  1069. set_bit(R5_WantFUA, &dev->flags);
  1070. if (sync)
  1071. set_bit(R5_SyncIO, &dev->flags);
  1072. }
  1073. }
  1074. if (sh->reconstruct_state == reconstruct_state_drain_run)
  1075. sh->reconstruct_state = reconstruct_state_drain_result;
  1076. else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)
  1077. sh->reconstruct_state = reconstruct_state_prexor_drain_result;
  1078. else {
  1079. BUG_ON(sh->reconstruct_state != reconstruct_state_run);
  1080. sh->reconstruct_state = reconstruct_state_result;
  1081. }
  1082. set_bit(STRIPE_HANDLE, &sh->state);
  1083. release_stripe(sh);
  1084. }
  1085. static void
  1086. ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
  1087. struct dma_async_tx_descriptor *tx)
  1088. {
  1089. int disks = sh->disks;
  1090. struct page **xor_srcs = percpu->scribble;
  1091. struct async_submit_ctl submit;
  1092. int count = 0, pd_idx = sh->pd_idx, i;
  1093. struct page *xor_dest;
  1094. int prexor = 0;
  1095. unsigned long flags;
  1096. pr_debug("%s: stripe %llu\n", __func__,
  1097. (unsigned long long)sh->sector);
  1098. for (i = 0; i < sh->disks; i++) {
  1099. if (pd_idx == i)
  1100. continue;
  1101. if (!test_bit(R5_Discard, &sh->dev[i].flags))
  1102. break;
  1103. }
  1104. if (i >= sh->disks) {
  1105. atomic_inc(&sh->count);
  1106. set_bit(R5_Discard, &sh->dev[pd_idx].flags);
  1107. ops_complete_reconstruct(sh);
  1108. return;
  1109. }
  1110. /* check if prexor is active which means only process blocks
  1111. * that are part of a read-modify-write (written)
  1112. */
  1113. if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
  1114. prexor = 1;
  1115. xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
  1116. for (i = disks; i--; ) {
  1117. struct r5dev *dev = &sh->dev[i];
  1118. if (dev->written)
  1119. xor_srcs[count++] = dev->page;
  1120. }
  1121. } else {
  1122. xor_dest = sh->dev[pd_idx].page;
  1123. for (i = disks; i--; ) {
  1124. struct r5dev *dev = &sh->dev[i];
  1125. if (i != pd_idx)
  1126. xor_srcs[count++] = dev->page;
  1127. }
  1128. }
  1129. /* 1/ if we prexor'd then the dest is reused as a source
  1130. * 2/ if we did not prexor then we are redoing the parity
  1131. * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST
  1132. * for the synchronous xor case
  1133. */
  1134. flags = ASYNC_TX_ACK |
  1135. (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST);
  1136. atomic_inc(&sh->count);
  1137. init_async_submit(&submit, flags, tx, ops_complete_reconstruct, sh,
  1138. to_addr_conv(sh, percpu));
  1139. if (unlikely(count == 1))
  1140. tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
  1141. else
  1142. tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
  1143. }
  1144. static void
  1145. ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
  1146. struct dma_async_tx_descriptor *tx)
  1147. {
  1148. struct async_submit_ctl submit;
  1149. struct page **blocks = percpu->scribble;
  1150. int count, i;
  1151. pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);
  1152. for (i = 0; i < sh->disks; i++) {
  1153. if (sh->pd_idx == i || sh->qd_idx == i)
  1154. continue;
  1155. if (!test_bit(R5_Discard, &sh->dev[i].flags))
  1156. break;
  1157. }
  1158. if (i >= sh->disks) {
  1159. atomic_inc(&sh->count);
  1160. set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
  1161. set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
  1162. ops_complete_reconstruct(sh);
  1163. return;
  1164. }
  1165. count = set_syndrome_sources(blocks, sh);
  1166. atomic_inc(&sh->count);
  1167. init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_reconstruct,
  1168. sh, to_addr_conv(sh, percpu));
  1169. async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
  1170. }
  1171. static void ops_complete_check(void *stripe_head_ref)
  1172. {
  1173. struct stripe_head *sh = stripe_head_ref;
  1174. pr_debug("%s: stripe %llu\n", __func__,
  1175. (unsigned long long)sh->sector);
  1176. sh->check_state = check_state_check_result;
  1177. set_bit(STRIPE_HANDLE, &sh->state);
  1178. release_stripe(sh);
  1179. }
  1180. static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
  1181. {
  1182. int disks = sh->disks;
  1183. int pd_idx = sh->pd_idx;
  1184. int qd_idx = sh->qd_idx;
  1185. struct page *xor_dest;
  1186. struct page **xor_srcs = percpu->scribble;
  1187. struct dma_async_tx_descriptor *tx;
  1188. struct async_submit_ctl submit;
  1189. int count;
  1190. int i;
  1191. pr_debug("%s: stripe %llu\n", __func__,
  1192. (unsigned long long)sh->sector);
  1193. count = 0;
  1194. xor_dest = sh->dev[pd_idx].page;
  1195. xor_srcs[count++] = xor_dest;
  1196. for (i = disks; i--; ) {
  1197. if (i == pd_idx || i == qd_idx)
  1198. continue;
  1199. xor_srcs[count++] = sh->dev[i].page;
  1200. }
  1201. init_async_submit(&submit, 0, NULL, NULL, NULL,
  1202. to_addr_conv(sh, percpu));
  1203. tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
  1204. &sh->ops.zero_sum_result, &submit);
  1205. atomic_inc(&sh->count);
  1206. init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);
  1207. tx = async_trigger_callback(&submit);
  1208. }
  1209. static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)
  1210. {
  1211. struct page **srcs = percpu->scribble;
  1212. struct async_submit_ctl submit;
  1213. int count;
  1214. pr_debug("%s: stripe %llu checkp: %d\n", __func__,
  1215. (unsigned long long)sh->sector, checkp);
  1216. count = set_syndrome_sources(srcs, sh);
  1217. if (!checkp)
  1218. srcs[count] = NULL;
  1219. atomic_inc(&sh->count);
  1220. init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,
  1221. sh, to_addr_conv(sh, percpu));
  1222. async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE,
  1223. &sh->ops.zero_sum_result, percpu->spare_page, &submit);
  1224. }
  1225. static void __raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
  1226. {
  1227. int overlap_clear = 0, i, disks = sh->disks;
  1228. struct dma_async_tx_descriptor *tx = NULL;
  1229. struct r5conf *conf = sh->raid_conf;
  1230. int level = conf->level;
  1231. struct raid5_percpu *percpu;
  1232. unsigned long cpu;
  1233. cpu = get_cpu();
  1234. percpu = per_cpu_ptr(conf->percpu, cpu);
  1235. if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
  1236. ops_run_biofill(sh);
  1237. overlap_clear++;
  1238. }
  1239. if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) {
  1240. if (level < 6)
  1241. tx = ops_run_compute5(sh, percpu);
  1242. else {
  1243. if (sh->ops.target2 < 0 || sh->ops.target < 0)
  1244. tx = ops_run_compute6_1(sh, percpu);
  1245. else
  1246. tx = ops_run_compute6_2(sh, percpu);
  1247. }
  1248. /* terminate the chain if reconstruct is not set to be run */
  1249. if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request))
  1250. async_tx_ack(tx);
  1251. }
  1252. if (test_bit(STRIPE_OP_PREXOR, &ops_request))
  1253. tx = ops_run_prexor(sh, percpu, tx);
  1254. if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) {
  1255. tx = ops_run_biodrain(sh, tx);
  1256. overlap_clear++;
  1257. }
  1258. if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) {
  1259. if (level < 6)
  1260. ops_run_reconstruct5(sh, percpu, tx);
  1261. else
  1262. ops_run_reconstruct6(sh, percpu, tx);
  1263. }
  1264. if (test_bit(STRIPE_OP_CHECK, &ops_request)) {
  1265. if (sh->check_state == check_state_run)
  1266. ops_run_check_p(sh, percpu);
  1267. else if (sh->check_state == check_state_run_q)
  1268. ops_run_check_pq(sh, percpu, 0);
  1269. else if (sh->check_state == check_state_run_pq)
  1270. ops_run_check_pq(sh, percpu, 1);
  1271. else
  1272. BUG();
  1273. }
  1274. if (overlap_clear)
  1275. for (i = disks; i--; ) {
  1276. struct r5dev *dev = &sh->dev[i];
  1277. if (test_and_clear_bit(R5_Overlap, &dev->flags))
  1278. wake_up(&sh->raid_conf->wait_for_overlap);
  1279. }
  1280. put_cpu();
  1281. }
  1282. #ifdef CONFIG_MULTICORE_RAID456
  1283. static void async_run_ops(void *param, async_cookie_t cookie)
  1284. {
  1285. struct stripe_head *sh = param;
  1286. unsigned long ops_request = sh->ops.request;
  1287. clear_bit_unlock(STRIPE_OPS_REQ_PENDING, &sh->state);
  1288. wake_up(&sh->ops.wait_for_ops);
  1289. __raid_run_ops(sh, ops_request);
  1290. release_stripe(sh);
  1291. }
  1292. static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
  1293. {
  1294. /* since handle_stripe can be called outside of raid5d context
  1295. * we need to ensure sh->ops.request is de-staged before another
  1296. * request arrives
  1297. */
  1298. wait_event(sh->ops.wait_for_ops,
  1299. !test_and_set_bit_lock(STRIPE_OPS_REQ_PENDING, &sh->state));
  1300. sh->ops.request = ops_request;
  1301. atomic_inc(&sh->count);
  1302. async_schedule(async_run_ops, sh);
  1303. }
  1304. #else
  1305. #define raid_run_ops __raid_run_ops
  1306. #endif
  1307. static int grow_one_stripe(struct r5conf *conf)
  1308. {
  1309. struct stripe_head *sh;
  1310. sh = kmem_cache_zalloc(conf->slab_cache, GFP_KERNEL);
  1311. if (!sh)
  1312. return 0;
  1313. sh->raid_conf = conf;
  1314. #ifdef CONFIG_MULTICORE_RAID456
  1315. init_waitqueue_head(&sh->ops.wait_for_ops);
  1316. #endif
  1317. spin_lock_init(&sh->stripe_lock);
  1318. if (grow_buffers(sh)) {
  1319. shrink_buffers(sh);
  1320. kmem_cache_free(conf->slab_cache, sh);
  1321. return 0;
  1322. }
  1323. /* we just created an active stripe so... */
  1324. atomic_set(&sh->count, 1);
  1325. atomic_inc(&conf->active_stripes);
  1326. INIT_LIST_HEAD(&sh->lru);
  1327. release_stripe(sh);
  1328. return 1;
  1329. }
  1330. static int grow_stripes(struct r5conf *conf, int num)
  1331. {
  1332. struct kmem_cache *sc;
  1333. int devs = max(conf->raid_disks, conf->previous_raid_disks);
  1334. if (conf->mddev->gendisk)
  1335. sprintf(conf->cache_name[0],
  1336. "raid%d-%s", conf->level, mdname(conf->mddev));
  1337. else
  1338. sprintf(conf->cache_name[0],
  1339. "raid%d-%p", conf->level, conf->mddev);
  1340. sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]);
  1341. conf->active_name = 0;
  1342. sc = kmem_cache_create(conf->cache_name[conf->active_name],
  1343. sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev),
  1344. 0, 0, NULL);
  1345. if (!sc)
  1346. return 1;
  1347. conf->slab_cache = sc;
  1348. conf->pool_size = devs;
  1349. while (num--)
  1350. if (!grow_one_stripe(conf))
  1351. return 1;
  1352. return 0;
  1353. }
  1354. /**
  1355. * scribble_len - return the required size of the scribble region
  1356. * @num - total number of disks in the array
  1357. *
  1358. * The size must be enough to contain:
  1359. * 1/ a struct page pointer for each device in the array +2
  1360. * 2/ room to convert each entry in (1) to its corresponding dma
  1361. * (dma_map_page()) or page (page_address()) address.
  1362. *
  1363. * Note: the +2 is for the destination buffers of the ddf/raid6 case where we
  1364. * calculate over all devices (not just the data blocks), using zeros in place
  1365. * of the P and Q blocks.
  1366. */
  1367. static size_t scribble_len(int num)
  1368. {
  1369. size_t len;
  1370. len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2);
  1371. return len;
  1372. }
  1373. static int resize_stripes(struct r5conf *conf, int newsize)
  1374. {
  1375. /* Make all the stripes able to hold 'newsize' devices.
  1376. * New slots in each stripe get 'page' set to a new page.
  1377. *
  1378. * This happens in stages:
  1379. * 1/ create a new kmem_cache and allocate the required number of
  1380. * stripe_heads.
  1381. * 2/ gather all the old stripe_heads and transfer the pages across
  1382. * to the new stripe_heads. This will have the side effect of
  1383. * freezing the array as once all stripe_heads have been collected,
  1384. * no IO will be possible. Old stripe heads are freed once their
  1385. * pages have been transferred over, and the old kmem_cache is
  1386. * freed when all stripes are done.
  1387. * 3/ reallocate conf->disks to be suitable bigger. If this fails,
  1388. * we simple return a failre status - no need to clean anything up.
  1389. * 4/ allocate new pages for the new slots in the new stripe_heads.
  1390. * If this fails, we don't bother trying the shrink the
  1391. * stripe_heads down again, we just leave them as they are.
  1392. * As each stripe_head is processed the new one is released into
  1393. * active service.
  1394. *
  1395. * Once step2 is started, we cannot afford to wait for a write,
  1396. * so we use GFP_NOIO allocations.
  1397. */
  1398. struct stripe_head *osh, *nsh;
  1399. LIST_HEAD(newstripes);
  1400. struct disk_info *ndisks;
  1401. unsigned long cpu;
  1402. int err;
  1403. struct kmem_cache *sc;
  1404. int i;
  1405. if (newsize <= conf->pool_size)
  1406. return 0; /* never bother to shrink */
  1407. err = md_allow_write(conf->mddev);
  1408. if (err)
  1409. return err;
  1410. /* Step 1 */
  1411. sc = kmem_cache_create(conf->cache_name[1-conf->active_name],
  1412. sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev),
  1413. 0, 0, NULL);
  1414. if (!sc)
  1415. return -ENOMEM;
  1416. for (i = conf->max_nr_stripes; i; i--) {
  1417. nsh = kmem_cache_zalloc(sc, GFP_KERNEL);
  1418. if (!nsh)
  1419. break;
  1420. nsh->raid_conf = conf;
  1421. #ifdef CONFIG_MULTICORE_RAID456
  1422. init_waitqueue_head(&nsh->ops.wait_for_ops);
  1423. #endif
  1424. spin_lock_init(&nsh->stripe_lock);
  1425. list_add(&nsh->lru, &newstripes);
  1426. }
  1427. if (i) {
  1428. /* didn't get enough, give up */
  1429. while (!list_empty(&newstripes)) {
  1430. nsh = list_entry(newstripes.next, struct stripe_head, lru);
  1431. list_del(&nsh->lru);
  1432. kmem_cache_free(sc, nsh);
  1433. }
  1434. kmem_cache_destroy(sc);
  1435. return -ENOMEM;
  1436. }
  1437. /* Step 2 - Must use GFP_NOIO now.
  1438. * OK, we have enough stripes, start collecting inactive
  1439. * stripes and copying them over
  1440. */
  1441. list_for_each_entry(nsh, &newstripes, lru) {
  1442. spin_lock_irq(&conf->device_lock);
  1443. wait_event_lock_irq(conf->wait_for_stripe,
  1444. !list_empty(&conf->inactive_list),
  1445. conf->device_lock);
  1446. osh = get_free_stripe(conf);
  1447. spin_unlock_irq(&conf->device_lock);
  1448. atomic_set(&nsh->count, 1);
  1449. for(i=0; i<conf->pool_size; i++)
  1450. nsh->dev[i].page = osh->dev[i].page;
  1451. for( ; i<newsize; i++)
  1452. nsh->dev[i].page = NULL;
  1453. kmem_cache_free(conf->slab_cache, osh);
  1454. }
  1455. kmem_cache_destroy(conf->slab_cache);
  1456. /* Step 3.
  1457. * At this point, we are holding all the stripes so the array
  1458. * is completely stalled, so now is a good time to resize
  1459. * conf->disks and the scribble region
  1460. */
  1461. ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO);
  1462. if (ndisks) {
  1463. for (i=0; i<conf->raid_disks; i++)
  1464. ndisks[i] = conf->disks[i];
  1465. kfree(conf->disks);
  1466. conf->disks = ndisks;
  1467. } else
  1468. err = -ENOMEM;
  1469. get_online_cpus();
  1470. conf->scribble_len = scribble_len(newsize);
  1471. for_each_present_cpu(cpu) {
  1472. struct raid5_percpu *percpu;
  1473. void *scribble;
  1474. percpu = per_cpu_ptr(conf->percpu, cpu);
  1475. scribble = kmalloc(conf->scribble_len, GFP_NOIO);
  1476. if (scribble) {
  1477. kfree(percpu->scribble);
  1478. percpu->scribble = scribble;
  1479. } else {
  1480. err = -ENOMEM;
  1481. break;
  1482. }
  1483. }
  1484. put_online_cpus();
  1485. /* Step 4, return new stripes to service */
  1486. while(!list_empty(&newstripes)) {
  1487. nsh = list_entry(newstripes.next, struct stripe_head, lru);
  1488. list_del_init(&nsh->lru);
  1489. for (i=conf->raid_disks; i < newsize; i++)
  1490. if (nsh->dev[i].page == NULL) {
  1491. struct page *p = alloc_page(GFP_NOIO);
  1492. nsh->dev[i].page = p;
  1493. if (!p)
  1494. err = -ENOMEM;
  1495. }
  1496. release_stripe(nsh);
  1497. }
  1498. /* critical section pass, GFP_NOIO no longer needed */
  1499. conf->slab_cache = sc;
  1500. conf->active_name = 1-conf->active_name;
  1501. conf->pool_size = newsize;
  1502. return err;
  1503. }
  1504. static int drop_one_stripe(struct r5conf *conf)
  1505. {
  1506. struct stripe_head *sh;
  1507. spin_lock_irq(&conf->device_lock);
  1508. sh = get_free_stripe(conf);
  1509. spin_unlock_irq(&conf->device_lock);
  1510. if (!sh)
  1511. return 0;
  1512. BUG_ON(atomic_read(&sh->count));
  1513. shrink_buffers(sh);
  1514. kmem_cache_free(conf->slab_cache, sh);
  1515. atomic_dec(&conf->active_stripes);
  1516. return 1;
  1517. }
  1518. static void shrink_stripes(struct r5conf *conf)
  1519. {
  1520. while (drop_one_stripe(conf))
  1521. ;
  1522. if (conf->slab_cache)
  1523. kmem_cache_destroy(conf->slab_cache);
  1524. conf->slab_cache = NULL;
  1525. }
  1526. static void raid5_end_read_request(struct bio * bi, int error)
  1527. {
  1528. struct stripe_head *sh = bi->bi_private;
  1529. struct r5conf *conf = sh->raid_conf;
  1530. int disks = sh->disks, i;
  1531. int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
  1532. char b[BDEVNAME_SIZE];
  1533. struct md_rdev *rdev = NULL;
  1534. sector_t s;
  1535. for (i=0 ; i<disks; i++)
  1536. if (bi == &sh->dev[i].req)
  1537. break;
  1538. pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n",
  1539. (unsigned long long)sh->sector, i, atomic_read(&sh->count),
  1540. uptodate);
  1541. if (i == disks) {
  1542. BUG();
  1543. return;
  1544. }
  1545. if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
  1546. /* If replacement finished while this request was outstanding,
  1547. * 'replacement' might be NULL already.
  1548. * In that case it moved down to 'rdev'.
  1549. * rdev is not removed until all requests are finished.
  1550. */
  1551. rdev = conf->disks[i].replacement;
  1552. if (!rdev)
  1553. rdev = conf->disks[i].rdev;
  1554. if (use_new_offset(conf, sh))
  1555. s = sh->sector + rdev->new_data_offset;
  1556. else
  1557. s = sh->sector + rdev->data_offset;
  1558. if (uptodate) {
  1559. set_bit(R5_UPTODATE, &sh->dev[i].flags);
  1560. if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
  1561. /* Note that this cannot happen on a
  1562. * replacement device. We just fail those on
  1563. * any error
  1564. */
  1565. printk_ratelimited(
  1566. KERN_INFO
  1567. "md/raid:%s: read error corrected"
  1568. " (%lu sectors at %llu on %s)\n",
  1569. mdname(conf->mddev), STRIPE_SECTORS,
  1570. (unsigned long long)s,
  1571. bdevname(rdev->bdev, b));
  1572. atomic_add(STRIPE_SECTORS, &rdev->corrected_errors);
  1573. clear_bit(R5_ReadError, &sh->dev[i].flags);
  1574. clear_bit(R5_ReWrite, &sh->dev[i].flags);
  1575. } else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
  1576. clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
  1577. if (atomic_read(&rdev->read_errors))
  1578. atomic_set(&rdev->read_errors, 0);
  1579. } else {
  1580. const char *bdn = bdevname(rdev->bdev, b);
  1581. int retry = 0;
  1582. int set_bad = 0;
  1583. clear_bit(R5_UPTODATE, &sh->dev[i].flags);
  1584. atomic_inc(&rdev->read_errors);
  1585. if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
  1586. printk_ratelimited(
  1587. KERN_WARNING
  1588. "md/raid:%s: read error on replacement device "
  1589. "(sector %llu on %s).\n",
  1590. mdname(conf->mddev),
  1591. (unsigned long long)s,
  1592. bdn);
  1593. else if (conf->mddev->degraded >= conf->max_degraded) {
  1594. set_bad = 1;
  1595. printk_ratelimited(
  1596. KERN_WARNING
  1597. "md/raid:%s: read error not correctable "
  1598. "(sector %llu on %s).\n",
  1599. mdname(conf->mddev),
  1600. (unsigned long long)s,
  1601. bdn);
  1602. } else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) {
  1603. /* Oh, no!!! */
  1604. set_bad = 1;
  1605. printk_ratelimited(
  1606. KERN_WARNING
  1607. "md/raid:%s: read error NOT corrected!! "
  1608. "(sector %llu on %s).\n",
  1609. mdname(conf->mddev),
  1610. (unsigned long long)s,
  1611. bdn);
  1612. } else if (atomic_read(&rdev->read_errors)
  1613. > conf->max_nr_stripes)
  1614. printk(KERN_WARNING
  1615. "md/raid:%s: Too many read errors, failing device %s.\n",
  1616. mdname(conf->mddev), bdn);
  1617. else
  1618. retry = 1;
  1619. if (retry)
  1620. if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) {
  1621. set_bit(R5_ReadError, &sh->dev[i].flags);
  1622. clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
  1623. } else
  1624. set_bit(R5_ReadNoMerge, &sh->dev[i].flags);
  1625. else {
  1626. clear_bit(R5_ReadError, &sh->dev[i].flags);
  1627. clear_bit(R5_ReWrite, &sh->dev[i].flags);
  1628. if (!(set_bad
  1629. && test_bit(In_sync, &rdev->flags)
  1630. && rdev_set_badblocks(
  1631. rdev, sh->sector, STRIPE_SECTORS, 0)))
  1632. md_error(conf->mddev, rdev);
  1633. }
  1634. }
  1635. rdev_dec_pending(rdev, conf->mddev);
  1636. clear_bit(R5_LOCKED, &sh->dev[i].flags);
  1637. set_bit(STRIPE_HANDLE, &sh->state);
  1638. release_stripe(sh);
  1639. }
  1640. static void raid5_end_write_request(struct bio *bi, int error)
  1641. {
  1642. struct stripe_head *sh = bi->bi_private;
  1643. struct r5conf *conf = sh->raid_conf;
  1644. int disks = sh->disks, i;
  1645. struct md_rdev *uninitialized_var(rdev);
  1646. int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
  1647. sector_t first_bad;
  1648. int bad_sectors;
  1649. int replacement = 0;
  1650. for (i = 0 ; i < disks; i++) {
  1651. if (bi == &sh->dev[i].req) {
  1652. rdev = conf->disks[i].rdev;
  1653. break;
  1654. }
  1655. if (bi == &sh->dev[i].rreq) {
  1656. rdev = conf->disks[i].replacement;
  1657. if (rdev)
  1658. replacement = 1;
  1659. else
  1660. /* rdev was removed and 'replacement'
  1661. * replaced it. rdev is not removed
  1662. * until all requests are finished.
  1663. */
  1664. rdev = conf->disks[i].rdev;
  1665. break;
  1666. }
  1667. }
  1668. pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n",
  1669. (unsigned long long)sh->sector, i, atomic_read(&sh->count),
  1670. uptodate);
  1671. if (i == disks) {
  1672. BUG();
  1673. return;
  1674. }
  1675. if (replacement) {
  1676. if (!uptodate)
  1677. md_error(conf->mddev, rdev);
  1678. else if (is_badblock(rdev, sh->sector,
  1679. STRIPE_SECTORS,
  1680. &first_bad, &bad_sectors))
  1681. set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);
  1682. } else {
  1683. if (!uptodate) {
  1684. set_bit(WriteErrorSeen, &rdev->flags);
  1685. set_bit(R5_WriteError, &sh->dev[i].flags);
  1686. if (!test_and_set_bit(WantReplacement, &rdev->flags))
  1687. set_bit(MD_RECOVERY_NEEDED,
  1688. &rdev->mddev->recovery);
  1689. } else if (is_badblock(rdev, sh->sector,
  1690. STRIPE_SECTORS,
  1691. &first_bad, &bad_sectors))
  1692. set_bit(R5_MadeGood, &sh->dev[i].flags);
  1693. }
  1694. rdev_dec_pending(rdev, conf->mddev);
  1695. if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))
  1696. clear_bit(R5_LOCKED, &sh->dev[i].flags);
  1697. set_bit(STRIPE_HANDLE, &sh->state);
  1698. release_stripe(sh);
  1699. }
  1700. static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous);
  1701. static void raid5_build_block(struct stripe_head *sh, int i, int previous)
  1702. {
  1703. struct r5dev *dev = &sh->dev[i];
  1704. bio_init(&dev->req);
  1705. dev->req.bi_io_vec = &dev->vec;
  1706. dev->req.bi_vcnt++;
  1707. dev->req.bi_max_vecs++;
  1708. dev->req.bi_private = sh;
  1709. dev->vec.bv_page = dev->page;
  1710. bio_init(&dev->rreq);
  1711. dev->rreq.bi_io_vec = &dev->rvec;
  1712. dev->rreq.bi_vcnt++;
  1713. dev->rreq.bi_max_vecs++;
  1714. dev->rreq.bi_private = sh;
  1715. dev->rvec.bv_page = dev->page;
  1716. dev->flags = 0;
  1717. dev->sector = compute_blocknr(sh, i, previous);
  1718. }
  1719. static void error(struct mddev *mddev, struct md_rdev *rdev)
  1720. {
  1721. char b[BDEVNAME_SIZE];
  1722. struct r5conf *conf = mddev->private;
  1723. unsigned long flags;
  1724. pr_debug("raid456: error called\n");
  1725. spin_lock_irqsave(&conf->device_lock, flags);
  1726. clear_bit(In_sync, &rdev->flags);
  1727. mddev->degraded = calc_degraded(conf);
  1728. spin_unlock_irqrestore(&conf->device_lock, flags);
  1729. set_bit(MD_RECOVERY_INTR, &mddev->recovery);
  1730. set_bit(Blocked, &rdev->flags);
  1731. set_bit(Faulty, &rdev->flags);
  1732. set_bit(MD_CHANGE_DEVS, &mddev->flags);
  1733. printk(KERN_ALERT
  1734. "md/raid:%s: Disk failure on %s, disabling device.\n"
  1735. "md/raid:%s: Operation continuing on %d devices.\n",
  1736. mdname(mddev),
  1737. bdevname(rdev->bdev, b),
  1738. mdname(mddev),
  1739. conf->raid_disks - mddev->degraded);
  1740. }
  1741. /*
  1742. * Input: a 'big' sector number,
  1743. * Output: index of the data and parity disk, and the sector # in them.
  1744. */
  1745. static sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector,
  1746. int previous, int *dd_idx,
  1747. struct stripe_head *sh)
  1748. {
  1749. sector_t stripe, stripe2;
  1750. sector_t chunk_number;
  1751. unsigned int chunk_offset;
  1752. int pd_idx, qd_idx;
  1753. int ddf_layout = 0;
  1754. sector_t new_sector;
  1755. int algorithm = previous ? conf->prev_algo
  1756. : conf->algorithm;
  1757. int sectors_per_chunk = previous ? conf->prev_chunk_sectors
  1758. : conf->chunk_sectors;
  1759. int raid_disks = previous ? conf->previous_raid_disks
  1760. : conf->raid_disks;
  1761. int data_disks = raid_disks - conf->max_degraded;
  1762. /* First compute the information on this sector */
  1763. /*
  1764. * Compute the chunk number and the sector offset inside the chunk
  1765. */
  1766. chunk_offset = sector_div(r_sector, sectors_per_chunk);
  1767. chunk_number = r_sector;
  1768. /*
  1769. * Compute the stripe number
  1770. */
  1771. stripe = chunk_number;
  1772. *dd_idx = sector_div(stripe, data_disks);
  1773. stripe2 = stripe;
  1774. /*
  1775. * Select the parity disk based on the user selected algorithm.
  1776. */
  1777. pd_idx = qd_idx = -1;
  1778. switch(conf->level) {
  1779. case 4:
  1780. pd_idx = data_disks;
  1781. break;
  1782. case 5:
  1783. switch (algorithm) {
  1784. case ALGORITHM_LEFT_ASYMMETRIC:
  1785. pd_idx = data_disks - sector_div(stripe2, raid_disks);
  1786. if (*dd_idx >= pd_idx)
  1787. (*dd_idx)++;
  1788. break;
  1789. case ALGORITHM_RIGHT_ASYMMETRIC:
  1790. pd_idx = sector_div(stripe2, raid_disks);
  1791. if (*dd_idx >= pd_idx)
  1792. (*dd_idx)++;
  1793. break;
  1794. case ALGORITHM_LEFT_SYMMETRIC:
  1795. pd_idx = data_disks - sector_div(stripe2, raid_disks);
  1796. *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
  1797. break;
  1798. case ALGORITHM_RIGHT_SYMMETRIC:
  1799. pd_idx = sector_div(stripe2, raid_disks);
  1800. *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
  1801. break;
  1802. case ALGORITHM_PARITY_0:
  1803. pd_idx = 0;
  1804. (*dd_idx)++;
  1805. break;
  1806. case ALGORITHM_PARITY_N:
  1807. pd_idx = data_disks;
  1808. break;
  1809. default:
  1810. BUG();
  1811. }
  1812. break;
  1813. case 6:
  1814. switch (algorithm) {
  1815. case ALGORITHM_LEFT_ASYMMETRIC:
  1816. pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
  1817. qd_idx = pd_idx + 1;
  1818. if (pd_idx == raid_disks-1) {
  1819. (*dd_idx)++; /* Q D D D P */
  1820. qd_idx = 0;
  1821. } else if (*dd_idx >= pd_idx)
  1822. (*dd_idx) += 2; /* D D P Q D */
  1823. break;
  1824. case ALGORITHM_RIGHT_ASYMMETRIC:
  1825. pd_idx = sector_div(stripe2, raid_disks);
  1826. qd_idx = pd_idx + 1;
  1827. if (pd_idx == raid_disks-1) {
  1828. (*dd_idx)++; /* Q D D D P */
  1829. qd_idx = 0;
  1830. } else if (*dd_idx >= pd_idx)
  1831. (*dd_idx) += 2; /* D D P Q D */
  1832. break;
  1833. case ALGORITHM_LEFT_SYMMETRIC:
  1834. pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
  1835. qd_idx = (pd_idx + 1) % raid_disks;
  1836. *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
  1837. break;
  1838. case ALGORITHM_RIGHT_SYMMETRIC:
  1839. pd_idx = sector_div(stripe2, raid_disks);
  1840. qd_idx = (pd_idx + 1) % raid_disks;
  1841. *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
  1842. break;
  1843. case ALGORITHM_PARITY_0:
  1844. pd_idx = 0;
  1845. qd_idx = 1;
  1846. (*dd_idx) += 2;
  1847. break;
  1848. case ALGORITHM_PARITY_N:
  1849. pd_idx = data_disks;
  1850. qd_idx = data_disks + 1;
  1851. break;
  1852. case ALGORITHM_ROTATING_ZERO_RESTART:
  1853. /* Exactly the same as RIGHT_ASYMMETRIC, but or
  1854. * of blocks for computing Q is different.
  1855. */
  1856. pd_idx = sector_div(stripe2, raid_disks);
  1857. qd_idx = pd_idx + 1;
  1858. if (pd_idx == raid_disks-1) {
  1859. (*dd_idx)++; /* Q D D D P */
  1860. qd_idx = 0;
  1861. } else if (*dd_idx >= pd_idx)
  1862. (*dd_idx) += 2; /* D D P Q D */
  1863. ddf_layout = 1;
  1864. break;
  1865. case ALGORITHM_ROTATING_N_RESTART:
  1866. /* Same a left_asymmetric, by first stripe is
  1867. * D D D P Q rather than
  1868. * Q D D D P
  1869. */
  1870. stripe2 += 1;
  1871. pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
  1872. qd_idx = pd_idx + 1;
  1873. if (pd_idx == raid_disks-1) {
  1874. (*dd_idx)++; /* Q D D D P */
  1875. qd_idx = 0;
  1876. } else if (*dd_idx >= pd_idx)
  1877. (*dd_idx) += 2; /* D D P Q D */
  1878. ddf_layout = 1;
  1879. break;
  1880. case ALGORITHM_ROTATING_N_CONTINUE:
  1881. /* Same as left_symmetric but Q is before P */
  1882. pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
  1883. qd_idx = (pd_idx + raid_disks - 1) % raid_disks;
  1884. *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
  1885. ddf_layout = 1;
  1886. break;
  1887. case ALGORITHM_LEFT_ASYMMETRIC_6:
  1888. /* RAID5 left_asymmetric, with Q on last device */
  1889. pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
  1890. if (*dd_idx >= pd_idx)
  1891. (*dd_idx)++;
  1892. qd_idx = raid_disks - 1;
  1893. break;
  1894. case ALGORITHM_RIGHT_ASYMMETRIC_6:
  1895. pd_idx = sector_div(stripe2, raid_disks-1);
  1896. if (*dd_idx >= pd_idx)
  1897. (*dd_idx)++;
  1898. qd_idx = raid_disks - 1;
  1899. break;
  1900. case ALGORITHM_LEFT_SYMMETRIC_6:
  1901. pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
  1902. *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
  1903. qd_idx = raid_disks - 1;
  1904. break;
  1905. case ALGORITHM_RIGHT_SYMMETRIC_6:
  1906. pd_idx = sector_div(stripe2, raid_disks-1);
  1907. *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
  1908. qd_idx = raid_disks - 1;
  1909. break;
  1910. case ALGORITHM_PARITY_0_6:
  1911. pd_idx = 0;
  1912. (*dd_idx)++;
  1913. qd_idx = raid_disks - 1;
  1914. break;
  1915. default:
  1916. BUG();
  1917. }
  1918. break;
  1919. }
  1920. if (sh) {
  1921. sh->pd_idx = pd_idx;
  1922. sh->qd_idx = qd_idx;
  1923. sh->ddf_layout = ddf_layout;
  1924. }
  1925. /*
  1926. * Finally, compute the new sector number
  1927. */
  1928. new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset;
  1929. return new_sector;
  1930. }
  1931. static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous)
  1932. {
  1933. struct r5conf *conf = sh->raid_conf;
  1934. int raid_disks = sh->disks;
  1935. int data_disks = raid_disks - conf->max_degraded;
  1936. sector_t new_sector = sh->sector, check;
  1937. int sectors_per_chunk = previous ? conf->prev_chunk_sectors
  1938. : conf->chunk_sectors;
  1939. int algorithm = previous ? conf->prev_algo
  1940. : conf->algorithm;
  1941. sector_t stripe;
  1942. int chunk_offset;
  1943. sector_t chunk_number;
  1944. int dummy1, dd_idx = i;
  1945. sector_t r_sector;
  1946. struct stripe_head sh2;
  1947. chunk_offset = sector_div(new_sector, sectors_per_chunk);
  1948. stripe = new_sector;
  1949. if (i == sh->pd_idx)
  1950. return 0;
  1951. switch(conf->level) {
  1952. case 4: break;
  1953. case 5:
  1954. switch (algorithm) {
  1955. case ALGORITHM_LEFT_ASYMMETRIC:
  1956. case ALGORITHM_RIGHT_ASYMMETRIC:
  1957. if (i > sh->pd_idx)
  1958. i--;
  1959. break;
  1960. case ALGORITHM_LEFT_SYMMETRIC:
  1961. case ALGORITHM_RIGHT_SYMMETRIC:
  1962. if (i < sh->pd_idx)
  1963. i += raid_disks;
  1964. i -= (sh->pd_idx + 1);
  1965. break;
  1966. case ALGORITHM_PARITY_0:
  1967. i -= 1;
  1968. break;
  1969. case ALGORITHM_PARITY_N:
  1970. break;
  1971. default:
  1972. BUG();
  1973. }
  1974. break;
  1975. case 6:
  1976. if (i == sh->qd_idx)
  1977. return 0; /* It is the Q disk */
  1978. switch (algorithm) {
  1979. case ALGORITHM_LEFT_ASYMMETRIC:
  1980. case ALGORITHM_RIGHT_ASYMMETRIC:
  1981. case ALGORITHM_ROTATING_ZERO_RESTART:
  1982. case ALGORITHM_ROTATING_N_RESTART:
  1983. if (sh->pd_idx == raid_disks-1)
  1984. i--; /* Q D D D P */
  1985. else if (i > sh->pd_idx)
  1986. i -= 2; /* D D P Q D */
  1987. break;
  1988. case ALGORITHM_LEFT_SYMMETRIC:
  1989. case ALGORITHM_RIGHT_SYMMETRIC:
  1990. if (sh->pd_idx == raid_disks-1)
  1991. i--; /* Q D D D P */
  1992. else {
  1993. /* D D P Q D */
  1994. if (i < sh->pd_idx)
  1995. i += raid_disks;
  1996. i -= (sh->pd_idx + 2);
  1997. }
  1998. break;
  1999. case ALGORITHM_PARITY_0:
  2000. i -= 2;
  2001. break;
  2002. case ALGORITHM_PARITY_N:
  2003. break;
  2004. case ALGORITHM_ROTATING_N_CONTINUE:
  2005. /* Like left_symmetric, but P is before Q */
  2006. if (sh->pd_idx == 0)
  2007. i--; /* P D D D Q */
  2008. else {
  2009. /* D D Q P D */
  2010. if (i < sh->pd_idx)
  2011. i += raid_disks;
  2012. i -= (sh->pd_idx + 1);
  2013. }
  2014. break;
  2015. case ALGORITHM_LEFT_ASYMMETRIC_6:
  2016. case ALGORITHM_RIGHT_ASYMMETRIC_6:
  2017. if (i > sh->pd_idx)
  2018. i--;
  2019. break;
  2020. case ALGORITHM_LEFT_SYMMETRIC_6:
  2021. case ALGORITHM_RIGHT_SYMMETRIC_6:
  2022. if (i < sh->pd_idx)
  2023. i += data_disks + 1;
  2024. i -= (sh->pd_idx + 1);
  2025. break;
  2026. case ALGORITHM_PARITY_0_6:
  2027. i -= 1;
  2028. break;
  2029. default:
  2030. BUG();
  2031. }
  2032. break;
  2033. }
  2034. chunk_number = stripe * data_disks + i;
  2035. r_sector = chunk_number * sectors_per_chunk + chunk_offset;
  2036. check = raid5_compute_sector(conf, r_sector,
  2037. previous, &dummy1, &sh2);
  2038. if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx
  2039. || sh2.qd_idx != sh->qd_idx) {
  2040. printk(KERN_ERR "md/raid:%s: compute_blocknr: map not correct\n",
  2041. mdname(conf->mddev));
  2042. return 0;
  2043. }
  2044. return r_sector;
  2045. }
  2046. static void
  2047. schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,
  2048. int rcw, int expand)
  2049. {
  2050. int i, pd_idx = sh->pd_idx, disks = sh->disks;
  2051. struct r5conf *conf = sh->raid_conf;
  2052. int level = conf->level;
  2053. if (rcw) {
  2054. /* if we are not expanding this is a proper write request, and
  2055. * there will be bios with new data to be drained into the
  2056. * stripe cache
  2057. */
  2058. if (!expand) {
  2059. sh->reconstruct_state = reconstruct_state_drain_run;
  2060. set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
  2061. } else
  2062. sh->reconstruct_state = reconstruct_state_run;
  2063. set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
  2064. for (i = disks; i--; ) {
  2065. struct r5dev *dev = &sh->dev[i];
  2066. if (dev->towrite) {
  2067. set_bit(R5_LOCKED, &dev->flags);
  2068. set_bit(R5_Wantdrain, &dev->flags);
  2069. if (!expand)
  2070. clear_bit(R5_UPTODATE, &dev->flags);
  2071. s->locked++;
  2072. }
  2073. }
  2074. if (s->locked + conf->max_degraded == disks)
  2075. if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))
  2076. atomic_inc(&conf->pending_full_writes);
  2077. } else {
  2078. BUG_ON(level == 6);
  2079. BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||
  2080. test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));
  2081. sh->reconstruct_state = reconstruct_state_prexor_drain_run;
  2082. set_bit(STRIPE_OP_PREXOR, &s->ops_request);
  2083. set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
  2084. set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
  2085. for (i = disks; i--; ) {
  2086. struct r5dev *dev = &sh->dev[i];
  2087. if (i == pd_idx)
  2088. continue;
  2089. if (dev->towrite &&
  2090. (test_bit(R5_UPTODATE, &dev->flags) ||
  2091. test_bit(R5_Wantcompute, &dev->flags))) {
  2092. set_bit(R5_Wantdrain, &dev->flags);
  2093. set_bit(R5_LOCKED, &dev->flags);
  2094. clear_bit(R5_UPTODATE, &dev->flags);
  2095. s->locked++;
  2096. }
  2097. }
  2098. }
  2099. /* keep the parity disk(s) locked while asynchronous operations
  2100. * are in flight
  2101. */
  2102. set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);
  2103. clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
  2104. s->locked++;
  2105. if (level == 6) {
  2106. int qd_idx = sh->qd_idx;
  2107. struct r5dev *dev = &sh->dev[qd_idx];
  2108. set_bit(R5_LOCKED, &dev->flags);
  2109. clear_bit(R5_UPTODATE, &dev->flags);
  2110. s->locked++;
  2111. }
  2112. pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n",
  2113. __func__, (unsigned long long)sh->sector,
  2114. s->locked, s->ops_request);
  2115. }
  2116. /*
  2117. * Each stripe/dev can have one or more bion attached.
  2118. * toread/towrite point to the first in a chain.
  2119. * The bi_next chain must be in order.
  2120. */
  2121. static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, int forwrite)
  2122. {
  2123. struct bio **bip;
  2124. struct r5conf *conf = sh->raid_conf;
  2125. int firstwrite=0;
  2126. pr_debug("adding bi b#%llu to stripe s#%llu\n",
  2127. (unsigned long long)bi->bi_sector,
  2128. (unsigned long long)sh->sector);
  2129. /*
  2130. * If several bio share a stripe. The bio bi_phys_segments acts as a
  2131. * reference count to avoid race. The reference count should already be
  2132. * increased before this function is called (for example, in
  2133. * make_request()), so other bio sharing this stripe will not free the
  2134. * stripe. If a stripe is owned by one stripe, the stripe lock will
  2135. * protect it.
  2136. */
  2137. spin_lock_irq(&sh->stripe_lock);
  2138. if (forwrite) {
  2139. bip = &sh->dev[dd_idx].towrite;
  2140. if (*bip == NULL)
  2141. firstwrite = 1;
  2142. } else
  2143. bip = &sh->dev[dd_idx].toread;
  2144. while (*bip && (*bip)->bi_sector < bi->bi_sector) {
  2145. if ((*bip)->bi_sector + ((*bip)->bi_size >> 9) > bi->bi_sector)
  2146. goto overlap;
  2147. bip = & (*bip)->bi_next;
  2148. }
  2149. if (*bip && (*bip)->bi_sector < bi->bi_sector + ((bi->bi_size)>>9))
  2150. goto overlap;
  2151. BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next);
  2152. if (*bip)
  2153. bi->bi_next = *bip;
  2154. *bip = bi;
  2155. raid5_inc_bi_active_stripes(bi);
  2156. if (forwrite) {
  2157. /* check if page is covered */
  2158. sector_t sector = sh->dev[dd_idx].sector;
  2159. for (bi=sh->dev[dd_idx].towrite;
  2160. sector < sh->dev[dd_idx].sector + STRIPE_SECTORS &&
  2161. bi && bi->bi_sector <= sector;
  2162. bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) {
  2163. if (bi->bi_sector + (bi->bi_size>>9) >= sector)
  2164. sector = bi->bi_sector + (bi->bi_size>>9);
  2165. }
  2166. if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS)
  2167. set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags);
  2168. }
  2169. pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n",
  2170. (unsigned long long)(*bip)->bi_sector,
  2171. (unsigned long long)sh->sector, dd_idx);
  2172. spin_unlock_irq(&sh->stripe_lock);
  2173. if (conf->mddev->bitmap && firstwrite) {
  2174. bitmap_startwrite(conf->mddev->bitmap, sh->sector,
  2175. STRIPE_SECTORS, 0);
  2176. sh->bm_seq = conf->seq_flush+1;
  2177. set_bit(STRIPE_BIT_DELAY, &sh->state);
  2178. }
  2179. return 1;
  2180. overlap:
  2181. set_bit(R5_Overlap, &sh->dev[dd_idx].flags);
  2182. spin_unlock_irq(&sh->stripe_lock);
  2183. return 0;
  2184. }
  2185. static void end_reshape(struct r5conf *conf);
  2186. static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
  2187. struct stripe_head *sh)
  2188. {
  2189. int sectors_per_chunk =
  2190. previous ? conf->prev_chunk_sectors : conf->chunk_sectors;
  2191. int dd_idx;
  2192. int chunk_offset = sector_div(stripe, sectors_per_chunk);
  2193. int disks = previous ? conf->previous_raid_disks : conf->raid_disks;
  2194. raid5_compute_sector(conf,
  2195. stripe * (disks - conf->max_degraded)
  2196. *sectors_per_chunk + chunk_offset,
  2197. previous,
  2198. &dd_idx, sh);
  2199. }
  2200. static void
  2201. handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,
  2202. struct stripe_head_state *s, int disks,
  2203. struct bio **return_bi)
  2204. {
  2205. int i;
  2206. for (i = disks; i--; ) {
  2207. struct bio *bi;
  2208. int bitmap_end = 0;
  2209. if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
  2210. struct md_rdev *rdev;
  2211. rcu_read_lock();
  2212. rdev = rcu_dereference(conf->disks[i].rdev);
  2213. if (rdev && test_bit(In_sync, &rdev->flags))
  2214. atomic_inc(&rdev->nr_pending);
  2215. else
  2216. rdev = NULL;
  2217. rcu_read_unlock();
  2218. if (rdev) {
  2219. if (!rdev_set_badblocks(
  2220. rdev,
  2221. sh->sector,
  2222. STRIPE_SECTORS, 0))
  2223. md_error(conf->mddev, rdev);
  2224. rdev_dec_pending(rdev, conf->mddev);
  2225. }
  2226. }
  2227. spin_lock_irq(&sh->stripe_lock);
  2228. /* fail all writes first */
  2229. bi = sh->dev[i].towrite;
  2230. sh->dev[i].towrite = NULL;
  2231. spin_unlock_irq(&sh->stripe_lock);
  2232. if (bi)
  2233. bitmap_end = 1;
  2234. if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
  2235. wake_up(&conf->wait_for_overlap);
  2236. while (bi && bi->bi_sector <
  2237. sh->dev[i].sector + STRIPE_SECTORS) {
  2238. struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);
  2239. clear_bit(BIO_UPTODATE, &bi->bi_flags);
  2240. if (!raid5_dec_bi_active_stripes(bi)) {
  2241. md_write_end(conf->mddev);
  2242. bi->bi_next = *return_bi;
  2243. *return_bi = bi;
  2244. }
  2245. bi = nextbi;
  2246. }
  2247. if (bitmap_end)
  2248. bitmap_endwrite(conf->mddev->bitmap, sh->sector,
  2249. STRIPE_SECTORS, 0, 0);
  2250. bitmap_end = 0;
  2251. /* and fail all 'written' */
  2252. bi = sh->dev[i].written;
  2253. sh->dev[i].written = NULL;
  2254. if (bi) bitmap_end = 1;
  2255. while (bi && bi->bi_sector <
  2256. sh->dev[i].sector + STRIPE_SECTORS) {
  2257. struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector);
  2258. clear_bit(BIO_UPTODATE, &bi->bi_flags);
  2259. if (!raid5_dec_bi_active_stripes(bi)) {
  2260. md_write_end(conf->mddev);
  2261. bi->bi_next = *return_bi;
  2262. *return_bi = bi;
  2263. }
  2264. bi = bi2;
  2265. }
  2266. /* fail any reads if this device is non-operational and
  2267. * the data has not reached the cache yet.
  2268. */
  2269. if (!test_bit(R5_Wantfill, &sh->dev[i].flags) &&
  2270. (!test_bit(R5_Insync, &sh->dev[i].flags) ||
  2271. test_bit(R5_ReadError, &sh->dev[i].flags))) {
  2272. spin_lock_irq(&sh->stripe_lock);
  2273. bi = sh->dev[i].toread;
  2274. sh->dev[i].toread = NULL;
  2275. spin_unlock_irq(&sh->stripe_lock);
  2276. if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
  2277. wake_up(&conf->wait_for_overlap);
  2278. while (bi && bi->bi_sector <
  2279. sh->dev[i].sector + STRIPE_SECTORS) {
  2280. struct bio *nextbi =
  2281. r5_next_bio(bi, sh->dev[i].sector);
  2282. clear_bit(BIO_UPTODATE, &bi->bi_flags);
  2283. if (!raid5_dec_bi_active_stripes(bi)) {
  2284. bi->bi_next = *return_bi;
  2285. *return_bi = bi;
  2286. }
  2287. bi = nextbi;
  2288. }
  2289. }
  2290. if (bitmap_end)
  2291. bitmap_endwrite(conf->mddev->bitmap, sh->sector,
  2292. STRIPE_SECTORS, 0, 0);
  2293. /* If we were in the middle of a write the parity block might
  2294. * still be locked - so just clear all R5_LOCKED flags
  2295. */
  2296. clear_bit(R5_LOCKED, &sh->dev[i].flags);
  2297. }
  2298. if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
  2299. if (atomic_dec_and_test(&conf->pending_full_writes))
  2300. md_wakeup_thread(conf->mddev->thread);
  2301. }
  2302. static void
  2303. handle_failed_sync(struct r5conf *conf, struct stripe_head *sh,
  2304. struct stripe_head_state *s)
  2305. {
  2306. int abort = 0;
  2307. int i;
  2308. clear_bit(STRIPE_SYNCING, &sh->state);
  2309. s->syncing = 0;
  2310. s->replacing = 0;
  2311. /* There is nothing more to do for sync/check/repair.
  2312. * Don't even need to abort as that is handled elsewhere
  2313. * if needed, and not always wanted e.g. if there is a known
  2314. * bad block here.
  2315. * For recover/replace we need to record a bad block on all
  2316. * non-sync devices, or abort the recovery
  2317. */
  2318. if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) {
  2319. /* During recovery devices cannot be removed, so
  2320. * locking and refcounting of rdevs is not needed
  2321. */
  2322. for (i = 0; i < conf->raid_disks; i++) {
  2323. struct md_rdev *rdev = conf->disks[i].rdev;
  2324. if (rdev
  2325. && !test_bit(Faulty, &rdev->flags)
  2326. && !test_bit(In_sync, &rdev->flags)
  2327. && !rdev_set_badblocks(rdev, sh->sector,
  2328. STRIPE_SECTORS, 0))
  2329. abort = 1;
  2330. rdev = conf->disks[i].replacement;
  2331. if (rdev
  2332. && !test_bit(Faulty, &rdev->flags)
  2333. && !test_bit(In_sync, &rdev->flags)
  2334. && !rdev_set_badblocks(rdev, sh->sector,
  2335. STRIPE_SECTORS, 0))
  2336. abort = 1;
  2337. }
  2338. if (abort)
  2339. conf->recovery_disabled =
  2340. conf->mddev->recovery_disabled;
  2341. }
  2342. md_done_sync(conf->mddev, STRIPE_SECTORS, !abort);
  2343. }
  2344. static int want_replace(struct stripe_head *sh, int disk_idx)
  2345. {
  2346. struct md_rdev *rdev;
  2347. int rv = 0;
  2348. /* Doing recovery so rcu locking not required */
  2349. rdev = sh->raid_conf->disks[disk_idx].replacement;
  2350. if (rdev
  2351. && !test_bit(Faulty, &rdev->flags)
  2352. && !test_bit(In_sync, &rdev->flags)
  2353. && (rdev->recovery_offset <= sh->sector
  2354. || rdev->mddev->recovery_cp <= sh->sector))
  2355. rv = 1;
  2356. return rv;
  2357. }
  2358. /* fetch_block - checks the given member device to see if its data needs
  2359. * to be read or computed to satisfy a request.
  2360. *
  2361. * Returns 1 when no more member devices need to be checked, otherwise returns
  2362. * 0 to tell the loop in handle_stripe_fill to continue
  2363. */
  2364. static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s,
  2365. int disk_idx, int disks)
  2366. {
  2367. struct r5dev *dev = &sh->dev[disk_idx];
  2368. struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]],
  2369. &sh->dev[s->failed_num[1]] };
  2370. /* is the data in this block needed, and can we get it? */
  2371. if (!test_bit(R5_LOCKED, &dev->flags) &&
  2372. !test_bit(R5_UPTODATE, &dev->flags) &&
  2373. (dev->toread ||
  2374. (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) ||
  2375. s->syncing || s->expanding ||
  2376. (s->replacing && want_replace(sh, disk_idx)) ||
  2377. (s->failed >= 1 && fdev[0]->toread) ||
  2378. (s->failed >= 2 && fdev[1]->toread) ||
  2379. (sh->raid_conf->level <= 5 && s->failed && fdev[0]->towrite &&
  2380. !test_bit(R5_OVERWRITE, &fdev[0]->flags)) ||
  2381. (sh->raid_conf->level == 6 && s->failed && s->to_write))) {
  2382. /* we would like to get this block, possibly by computing it,
  2383. * otherwise read it if the backing disk is insync
  2384. */
  2385. BUG_ON(test_bit(R5_Wantcompute, &dev->flags));
  2386. BUG_ON(test_bit(R5_Wantread, &dev->flags));
  2387. if ((s->uptodate == disks - 1) &&
  2388. (s->failed && (disk_idx == s->failed_num[0] ||
  2389. disk_idx == s->failed_num[1]))) {
  2390. /* have disk failed, and we're requested to fetch it;
  2391. * do compute it
  2392. */
  2393. pr_debug("Computing stripe %llu block %d\n",
  2394. (unsigned long long)sh->sector, disk_idx);
  2395. set_bit(STRIPE_COMPUTE_RUN, &sh->state);
  2396. set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
  2397. set_bit(R5_Wantcompute, &dev->flags);
  2398. sh->ops.target = disk_idx;
  2399. sh->ops.target2 = -1; /* no 2nd target */
  2400. s->req_compute = 1;
  2401. /* Careful: from this point on 'uptodate' is in the eye
  2402. * of raid_run_ops which services 'compute' operations
  2403. * before writes. R5_Wantcompute flags a block that will
  2404. * be R5_UPTODATE by the time it is needed for a
  2405. * subsequent operation.
  2406. */
  2407. s->uptodate++;
  2408. return 1;
  2409. } else if (s->uptodate == disks-2 && s->failed >= 2) {
  2410. /* Computing 2-failure is *very* expensive; only
  2411. * do it if failed >= 2
  2412. */
  2413. int other;
  2414. for (other = disks; other--; ) {
  2415. if (other == disk_idx)
  2416. continue;
  2417. if (!test_bit(R5_UPTODATE,
  2418. &sh->dev[other].flags))
  2419. break;
  2420. }
  2421. BUG_ON(other < 0);
  2422. pr_debug("Computing stripe %llu blocks %d,%d\n",
  2423. (unsigned long long)sh->sector,
  2424. disk_idx, other);
  2425. set_bit(STRIPE_COMPUTE_RUN, &sh->state);
  2426. set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
  2427. set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags);
  2428. set_bit(R5_Wantcompute, &sh->dev[other].flags);
  2429. sh->ops.target = disk_idx;
  2430. sh->ops.target2 = other;
  2431. s->uptodate += 2;
  2432. s->req_compute = 1;
  2433. return 1;
  2434. } else if (test_bit(R5_Insync, &dev->flags)) {
  2435. set_bit(R5_LOCKED, &dev->flags);
  2436. set_bit(R5_Wantread, &dev->flags);
  2437. s->locked++;
  2438. pr_debug("Reading block %d (sync=%d)\n",
  2439. disk_idx, s->syncing);
  2440. }
  2441. }
  2442. return 0;
  2443. }
  2444. /**
  2445. * handle_stripe_fill - read or compute data to satisfy pending requests.
  2446. */
  2447. static void handle_stripe_fill(struct stripe_head *sh,
  2448. struct stripe_head_state *s,
  2449. int disks)
  2450. {
  2451. int i;
  2452. /* look for blocks to read/compute, skip this if a compute
  2453. * is already in flight, or if the stripe contents are in the
  2454. * midst of changing due to a write
  2455. */
  2456. if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&
  2457. !sh->reconstruct_state)
  2458. for (i = disks; i--; )
  2459. if (fetch_block(sh, s, i, disks))
  2460. break;
  2461. set_bit(STRIPE_HANDLE, &sh->state);
  2462. }
  2463. /* handle_stripe_clean_event
  2464. * any written block on an uptodate or failed drive can be returned.
  2465. * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but
  2466. * never LOCKED, so we don't need to test 'failed' directly.
  2467. */
  2468. static void handle_stripe_clean_event(struct r5conf *conf,
  2469. struct stripe_head *sh, int disks, struct bio **return_bi)
  2470. {
  2471. int i;
  2472. struct r5dev *dev;
  2473. for (i = disks; i--; )
  2474. if (sh->dev[i].written) {
  2475. dev = &sh->dev[i];
  2476. if (!test_bit(R5_LOCKED, &dev->flags) &&
  2477. (test_bit(R5_UPTODATE, &dev->flags) ||
  2478. test_bit(R5_Discard, &dev->flags))) {
  2479. /* We can return any write requests */
  2480. struct bio *wbi, *wbi2;
  2481. pr_debug("Return write for disc %d\n", i);
  2482. if (test_and_clear_bit(R5_Discard, &dev->flags))
  2483. clear_bit(R5_UPTODATE, &dev->flags);
  2484. wbi = dev->written;
  2485. dev->written = NULL;
  2486. while (wbi && wbi->bi_sector <
  2487. dev->sector + STRIPE_SECTORS) {
  2488. wbi2 = r5_next_bio(wbi, dev->sector);
  2489. if (!raid5_dec_bi_active_stripes(wbi)) {
  2490. md_write_end(conf->mddev);
  2491. wbi->bi_next = *return_bi;
  2492. *return_bi = wbi;
  2493. }
  2494. wbi = wbi2;
  2495. }
  2496. bitmap_endwrite(conf->mddev->bitmap, sh->sector,
  2497. STRIPE_SECTORS,
  2498. !test_bit(STRIPE_DEGRADED, &sh->state),
  2499. 0);
  2500. }
  2501. } else if (test_bit(R5_Discard, &sh->dev[i].flags))
  2502. clear_bit(R5_Discard, &sh->dev[i].flags);
  2503. if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
  2504. if (atomic_dec_and_test(&conf->pending_full_writes))
  2505. md_wakeup_thread(conf->mddev->thread);
  2506. }
  2507. static void handle_stripe_dirtying(struct r5conf *conf,
  2508. struct stripe_head *sh,
  2509. struct stripe_head_state *s,
  2510. int disks)
  2511. {
  2512. int rmw = 0, rcw = 0, i;
  2513. sector_t recovery_cp = conf->mddev->recovery_cp;
  2514. /* RAID6 requires 'rcw' in current implementation.
  2515. * Otherwise, check whether resync is now happening or should start.
  2516. * If yes, then the array is dirty (after unclean shutdown or
  2517. * initial creation), so parity in some stripes might be inconsistent.
  2518. * In this case, we need to always do reconstruct-write, to ensure
  2519. * that in case of drive failure or read-error correction, we
  2520. * generate correct data from the parity.
  2521. */
  2522. if (conf->max_degraded == 2 ||
  2523. (recovery_cp < MaxSector && sh->sector >= recovery_cp)) {
  2524. /* Calculate the real rcw later - for now make it
  2525. * look like rcw is cheaper
  2526. */
  2527. rcw = 1; rmw = 2;
  2528. pr_debug("force RCW max_degraded=%u, recovery_cp=%llu sh->sector=%llu\n",
  2529. conf->max_degraded, (unsigned long long)recovery_cp,
  2530. (unsigned long long)sh->sector);
  2531. } else for (i = disks; i--; ) {
  2532. /* would I have to read this buffer for read_modify_write */
  2533. struct r5dev *dev = &sh->dev[i];
  2534. if ((dev->towrite || i == sh->pd_idx) &&
  2535. !test_bit(R5_LOCKED, &dev->flags) &&
  2536. !(test_bit(R5_UPTODATE, &dev->flags) ||
  2537. test_bit(R5_Wantcompute, &dev->flags))) {
  2538. if (test_bit(R5_Insync, &dev->flags))
  2539. rmw++;
  2540. else
  2541. rmw += 2*disks; /* cannot read it */
  2542. }
  2543. /* Would I have to read this buffer for reconstruct_write */
  2544. if (!test_bit(R5_OVERWRITE, &dev->flags) && i != sh->pd_idx &&
  2545. !test_bit(R5_LOCKED, &dev->flags) &&
  2546. !(test_bit(R5_UPTODATE, &dev->flags) ||
  2547. test_bit(R5_Wantcompute, &dev->flags))) {
  2548. if (test_bit(R5_Insync, &dev->flags)) rcw++;
  2549. else
  2550. rcw += 2*disks;
  2551. }
  2552. }
  2553. pr_debug("for sector %llu, rmw=%d rcw=%d\n",
  2554. (unsigned long long)sh->sector, rmw, rcw);
  2555. set_bit(STRIPE_HANDLE, &sh->state);
  2556. if (rmw < rcw && rmw > 0) {
  2557. /* prefer read-modify-write, but need to get some data */
  2558. blk_add_trace_msg(conf->mddev->queue, "raid5 rmw %llu %d",
  2559. (unsigned long long)sh->sector, rmw);
  2560. for (i = disks; i--; ) {
  2561. struct r5dev *dev = &sh->dev[i];
  2562. if ((dev->towrite || i == sh->pd_idx) &&
  2563. !test_bit(R5_LOCKED, &dev->flags) &&
  2564. !(test_bit(R5_UPTODATE, &dev->flags) ||
  2565. test_bit(R5_Wantcompute, &dev->flags)) &&
  2566. test_bit(R5_Insync, &dev->flags)) {
  2567. if (
  2568. test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
  2569. pr_debug("Read_old block "
  2570. "%d for r-m-w\n", i);
  2571. set_bit(R5_LOCKED, &dev->flags);
  2572. set_bit(R5_Wantread, &dev->flags);
  2573. s->locked++;
  2574. } else {
  2575. set_bit(STRIPE_DELAYED, &sh->state);
  2576. set_bit(STRIPE_HANDLE, &sh->state);
  2577. }
  2578. }
  2579. }
  2580. }
  2581. if (rcw <= rmw && rcw > 0) {
  2582. /* want reconstruct write, but need to get some data */
  2583. int qread =0;
  2584. rcw = 0;
  2585. for (i = disks; i--; ) {
  2586. struct r5dev *dev = &sh->dev[i];
  2587. if (!test_bit(R5_OVERWRITE, &dev->flags) &&
  2588. i != sh->pd_idx && i != sh->qd_idx &&
  2589. !test_bit(R5_LOCKED, &dev->flags) &&
  2590. !(test_bit(R5_UPTODATE, &dev->flags) ||
  2591. test_bit(R5_Wantcompute, &dev->flags))) {
  2592. rcw++;
  2593. if (!test_bit(R5_Insync, &dev->flags))
  2594. continue; /* it's a failed drive */
  2595. if (
  2596. test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
  2597. pr_debug("Read_old block "
  2598. "%d for Reconstruct\n", i);
  2599. set_bit(R5_LOCKED, &dev->flags);
  2600. set_bit(R5_Wantread, &dev->flags);
  2601. s->locked++;
  2602. qread++;
  2603. } else {
  2604. set_bit(STRIPE_DELAYED, &sh->state);
  2605. set_bit(STRIPE_HANDLE, &sh->state);
  2606. }
  2607. }
  2608. }
  2609. if (rcw)
  2610. blk_add_trace_msg(conf->mddev->queue, "raid5 rcw %llu %d %d %d",
  2611. (unsigned long long)sh->sector,
  2612. rcw, qread, test_bit(STRIPE_DELAYED, &sh->state));
  2613. }
  2614. /* now if nothing is locked, and if we have enough data,
  2615. * we can start a write request
  2616. */
  2617. /* since handle_stripe can be called at any time we need to handle the
  2618. * case where a compute block operation has been submitted and then a
  2619. * subsequent call wants to start a write request. raid_run_ops only
  2620. * handles the case where compute block and reconstruct are requested
  2621. * simultaneously. If this is not the case then new writes need to be
  2622. * held off until the compute completes.
  2623. */
  2624. if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
  2625. (s->locked == 0 && (rcw == 0 || rmw == 0) &&
  2626. !test_bit(STRIPE_BIT_DELAY, &sh->state)))
  2627. schedule_reconstruction(sh, s, rcw == 0, 0);
  2628. }
  2629. static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh,
  2630. struct stripe_head_state *s, int disks)
  2631. {
  2632. struct r5dev *dev = NULL;
  2633. set_bit(STRIPE_HANDLE, &sh->state);
  2634. switch (sh->check_state) {
  2635. case check_state_idle:
  2636. /* start a new check operation if there are no failures */
  2637. if (s->failed == 0) {
  2638. BUG_ON(s->uptodate != disks);
  2639. sh->check_state = check_state_run;
  2640. set_bit(STRIPE_OP_CHECK, &s->ops_request);
  2641. clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
  2642. s->uptodate--;
  2643. break;
  2644. }
  2645. dev = &sh->dev[s->failed_num[0]];
  2646. /* fall through */
  2647. case check_state_compute_result:
  2648. sh->check_state = check_state_idle;
  2649. if (!dev)
  2650. dev = &sh->dev[sh->pd_idx];
  2651. /* check that a write has not made the stripe insync */
  2652. if (test_bit(STRIPE_INSYNC, &sh->state))
  2653. break;
  2654. /* either failed parity check, or recovery is happening */
  2655. BUG_ON(!test_bit(R5_UPTODATE, &dev->flags));
  2656. BUG_ON(s->uptodate != disks);
  2657. set_bit(R5_LOCKED, &dev->flags);
  2658. s->locked++;
  2659. set_bit(R5_Wantwrite, &dev->flags);
  2660. clear_bit(STRIPE_DEGRADED, &sh->state);
  2661. set_bit(STRIPE_INSYNC, &sh->state);
  2662. break;
  2663. case check_state_run:
  2664. break; /* we will be called again upon completion */
  2665. case check_state_check_result:
  2666. sh->check_state = check_state_idle;
  2667. /* if a failure occurred during the check operation, leave
  2668. * STRIPE_INSYNC not set and let the stripe be handled again
  2669. */
  2670. if (s->failed)
  2671. break;
  2672. /* handle a successful check operation, if parity is correct
  2673. * we are done. Otherwise update the mismatch count and repair
  2674. * parity if !MD_RECOVERY_CHECK
  2675. */
  2676. if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0)
  2677. /* parity is correct (on disc,
  2678. * not in buffer any more)
  2679. */
  2680. set_bit(STRIPE_INSYNC, &sh->state);
  2681. else {
  2682. atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches);
  2683. if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
  2684. /* don't try to repair!! */
  2685. set_bit(STRIPE_INSYNC, &sh->state);
  2686. else {
  2687. sh->check_state = check_state_compute_run;
  2688. set_bit(STRIPE_COMPUTE_RUN, &sh->state);
  2689. set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
  2690. set_bit(R5_Wantcompute,
  2691. &sh->dev[sh->pd_idx].flags);
  2692. sh->ops.target = sh->pd_idx;
  2693. sh->ops.target2 = -1;
  2694. s->uptodate++;
  2695. }
  2696. }
  2697. break;
  2698. case check_state_compute_run:
  2699. break;
  2700. default:
  2701. printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
  2702. __func__, sh->check_state,
  2703. (unsigned long long) sh->sector);
  2704. BUG();
  2705. }
  2706. }
  2707. static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh,
  2708. struct stripe_head_state *s,
  2709. int disks)
  2710. {
  2711. int pd_idx = sh->pd_idx;
  2712. int qd_idx = sh->qd_idx;
  2713. struct r5dev *dev;
  2714. set_bit(STRIPE_HANDLE, &sh->state);
  2715. BUG_ON(s->failed > 2);
  2716. /* Want to check and possibly repair P and Q.
  2717. * However there could be one 'failed' device, in which
  2718. * case we can only check one of them, possibly using the
  2719. * other to generate missing data
  2720. */
  2721. switch (sh->check_state) {
  2722. case check_state_idle:
  2723. /* start a new check operation if there are < 2 failures */
  2724. if (s->failed == s->q_failed) {
  2725. /* The only possible failed device holds Q, so it
  2726. * makes sense to check P (If anything else were failed,
  2727. * we would have used P to recreate it).
  2728. */
  2729. sh->check_state = check_state_run;
  2730. }
  2731. if (!s->q_failed && s->failed < 2) {
  2732. /* Q is not failed, and we didn't use it to generate
  2733. * anything, so it makes sense to check it
  2734. */
  2735. if (sh->check_state == check_state_run)
  2736. sh->check_state = check_state_run_pq;
  2737. else
  2738. sh->check_state = check_state_run_q;
  2739. }
  2740. /* discard potentially stale zero_sum_result */
  2741. sh->ops.zero_sum_result = 0;
  2742. if (sh->check_state == check_state_run) {
  2743. /* async_xor_zero_sum destroys the contents of P */
  2744. clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
  2745. s->uptodate--;
  2746. }
  2747. if (sh->check_state >= check_state_run &&
  2748. sh->check_state <= check_state_run_pq) {
  2749. /* async_syndrome_zero_sum preserves P and Q, so
  2750. * no need to mark them !uptodate here
  2751. */
  2752. set_bit(STRIPE_OP_CHECK, &s->ops_request);
  2753. break;
  2754. }
  2755. /* we have 2-disk failure */
  2756. BUG_ON(s->failed != 2);
  2757. /* fall through */
  2758. case check_state_compute_result:
  2759. sh->check_state = check_state_idle;
  2760. /* check that a write has not made the stripe insync */
  2761. if (test_bit(STRIPE_INSYNC, &sh->state))
  2762. break;
  2763. /* now write out any block on a failed drive,
  2764. * or P or Q if they were recomputed
  2765. */
  2766. BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */
  2767. if (s->failed == 2) {
  2768. dev = &sh->dev[s->failed_num[1]];
  2769. s->locked++;
  2770. set_bit(R5_LOCKED, &dev->flags);
  2771. set_bit(R5_Wantwrite, &dev->flags);
  2772. }
  2773. if (s->failed >= 1) {
  2774. dev = &sh->dev[s->failed_num[0]];
  2775. s->locked++;
  2776. set_bit(R5_LOCKED, &dev->flags);
  2777. set_bit(R5_Wantwrite, &dev->flags);
  2778. }
  2779. if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
  2780. dev = &sh->dev[pd_idx];
  2781. s->locked++;
  2782. set_bit(R5_LOCKED, &dev->flags);
  2783. set_bit(R5_Wantwrite, &dev->flags);
  2784. }
  2785. if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
  2786. dev = &sh->dev[qd_idx];
  2787. s->locked++;
  2788. set_bit(R5_LOCKED, &dev->flags);
  2789. set_bit(R5_Wantwrite, &dev->flags);
  2790. }
  2791. clear_bit(STRIPE_DEGRADED, &sh->state);
  2792. set_bit(STRIPE_INSYNC, &sh->state);
  2793. break;
  2794. case check_state_run:
  2795. case check_state_run_q:
  2796. case check_state_run_pq:
  2797. break; /* we will be called again upon completion */
  2798. case check_state_check_result:
  2799. sh->check_state = check_state_idle;
  2800. /* handle a successful check operation, if parity is correct
  2801. * we are done. Otherwise update the mismatch count and repair
  2802. * parity if !MD_RECOVERY_CHECK
  2803. */
  2804. if (sh->ops.zero_sum_result == 0) {
  2805. /* both parities are correct */
  2806. if (!s->failed)
  2807. set_bit(STRIPE_INSYNC, &sh->state);
  2808. else {
  2809. /* in contrast to the raid5 case we can validate
  2810. * parity, but still have a failure to write
  2811. * back
  2812. */
  2813. sh->check_state = check_state_compute_result;
  2814. /* Returning at this point means that we may go
  2815. * off and bring p and/or q uptodate again so
  2816. * we make sure to check zero_sum_result again
  2817. * to verify if p or q need writeback
  2818. */
  2819. }
  2820. } else {
  2821. atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches);
  2822. if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
  2823. /* don't try to repair!! */
  2824. set_bit(STRIPE_INSYNC, &sh->state);
  2825. else {
  2826. int *target = &sh->ops.target;
  2827. sh->ops.target = -1;
  2828. sh->ops.target2 = -1;
  2829. sh->check_state = check_state_compute_run;
  2830. set_bit(STRIPE_COMPUTE_RUN, &sh->state);
  2831. set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
  2832. if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
  2833. set_bit(R5_Wantcompute,
  2834. &sh->dev[pd_idx].flags);
  2835. *target = pd_idx;
  2836. target = &sh->ops.target2;
  2837. s->uptodate++;
  2838. }
  2839. if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
  2840. set_bit(R5_Wantcompute,
  2841. &sh->dev[qd_idx].flags);
  2842. *target = qd_idx;
  2843. s->uptodate++;
  2844. }
  2845. }
  2846. }
  2847. break;
  2848. case check_state_compute_run:
  2849. break;
  2850. default:
  2851. printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
  2852. __func__, sh->check_state,
  2853. (unsigned long long) sh->sector);
  2854. BUG();
  2855. }
  2856. }
  2857. static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)
  2858. {
  2859. int i;
  2860. /* We have read all the blocks in this stripe and now we need to
  2861. * copy some of them into a target stripe for expand.
  2862. */
  2863. struct dma_async_tx_descriptor *tx = NULL;
  2864. clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);
  2865. for (i = 0; i < sh->disks; i++)
  2866. if (i != sh->pd_idx && i != sh->qd_idx) {
  2867. int dd_idx, j;
  2868. struct stripe_head *sh2;
  2869. struct async_submit_ctl submit;
  2870. sector_t bn = compute_blocknr(sh, i, 1);
  2871. sector_t s = raid5_compute_sector(conf, bn, 0,
  2872. &dd_idx, NULL);
  2873. sh2 = get_active_stripe(conf, s, 0, 1, 1);
  2874. if (sh2 == NULL)
  2875. /* so far only the early blocks of this stripe
  2876. * have been requested. When later blocks
  2877. * get requested, we will try again
  2878. */
  2879. continue;
  2880. if (!test_bit(STRIPE_EXPANDING, &sh2->state) ||
  2881. test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) {
  2882. /* must have already done this block */
  2883. release_stripe(sh2);
  2884. continue;
  2885. }
  2886. /* place all the copies on one channel */
  2887. init_async_submit(&submit, 0, tx, NULL, NULL, NULL);
  2888. tx = async_memcpy(sh2->dev[dd_idx].page,
  2889. sh->dev[i].page, 0, 0, STRIPE_SIZE,
  2890. &submit);
  2891. set_bit(R5_Expanded, &sh2->dev[dd_idx].flags);
  2892. set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags);
  2893. for (j = 0; j < conf->raid_disks; j++)
  2894. if (j != sh2->pd_idx &&
  2895. j != sh2->qd_idx &&
  2896. !test_bit(R5_Expanded, &sh2->dev[j].flags))
  2897. break;
  2898. if (j == conf->raid_disks) {
  2899. set_bit(STRIPE_EXPAND_READY, &sh2->state);
  2900. set_bit(STRIPE_HANDLE, &sh2->state);
  2901. }
  2902. release_stripe(sh2);
  2903. }
  2904. /* done submitting copies, wait for them to complete */
  2905. async_tx_quiesce(&tx);
  2906. }
  2907. /*
  2908. * handle_stripe - do things to a stripe.
  2909. *
  2910. * We lock the stripe by setting STRIPE_ACTIVE and then examine the
  2911. * state of various bits to see what needs to be done.
  2912. * Possible results:
  2913. * return some read requests which now have data
  2914. * return some write requests which are safely on storage
  2915. * schedule a read on some buffers
  2916. * schedule a write of some buffers
  2917. * return confirmation of parity correctness
  2918. *
  2919. */
  2920. static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
  2921. {
  2922. struct r5conf *conf = sh->raid_conf;
  2923. int disks = sh->disks;
  2924. struct r5dev *dev;
  2925. int i;
  2926. int do_recovery = 0;
  2927. memset(s, 0, sizeof(*s));
  2928. s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state);
  2929. s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state);
  2930. s->failed_num[0] = -1;
  2931. s->failed_num[1] = -1;
  2932. /* Now to look around and see what can be done */
  2933. rcu_read_lock();
  2934. for (i=disks; i--; ) {
  2935. struct md_rdev *rdev;
  2936. sector_t first_bad;
  2937. int bad_sectors;
  2938. int is_bad = 0;
  2939. dev = &sh->dev[i];
  2940. pr_debug("check %d: state 0x%lx read %p write %p written %p\n",
  2941. i, dev->flags,
  2942. dev->toread, dev->towrite, dev->written);
  2943. /* maybe we can reply to a read
  2944. *
  2945. * new wantfill requests are only permitted while
  2946. * ops_complete_biofill is guaranteed to be inactive
  2947. */
  2948. if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread &&
  2949. !test_bit(STRIPE_BIOFILL_RUN, &sh->state))
  2950. set_bit(R5_Wantfill, &dev->flags);
  2951. /* now count some things */
  2952. if (test_bit(R5_LOCKED, &dev->flags))
  2953. s->locked++;
  2954. if (test_bit(R5_UPTODATE, &dev->flags))
  2955. s->uptodate++;
  2956. if (test_bit(R5_Wantcompute, &dev->flags)) {
  2957. s->compute++;
  2958. BUG_ON(s->compute > 2);
  2959. }
  2960. if (test_bit(R5_Wantfill, &dev->flags))
  2961. s->to_fill++;
  2962. else if (dev->toread)
  2963. s->to_read++;
  2964. if (dev->towrite) {
  2965. s->to_write++;
  2966. if (!test_bit(R5_OVERWRITE, &dev->flags))
  2967. s->non_overwrite++;
  2968. }
  2969. if (dev->written)
  2970. s->written++;
  2971. /* Prefer to use the replacement for reads, but only
  2972. * if it is recovered enough and has no bad blocks.
  2973. */
  2974. rdev = rcu_dereference(conf->disks[i].replacement);
  2975. if (rdev && !test_bit(Faulty, &rdev->flags) &&
  2976. rdev->recovery_offset >= sh->sector + STRIPE_SECTORS &&
  2977. !is_badblock(rdev, sh->sector, STRIPE_SECTORS,
  2978. &first_bad, &bad_sectors))
  2979. set_bit(R5_ReadRepl, &dev->flags);
  2980. else {
  2981. if (rdev)
  2982. set_bit(R5_NeedReplace, &dev->flags);
  2983. rdev = rcu_dereference(conf->disks[i].rdev);
  2984. clear_bit(R5_ReadRepl, &dev->flags);
  2985. }
  2986. if (rdev && test_bit(Faulty, &rdev->flags))
  2987. rdev = NULL;
  2988. if (rdev) {
  2989. is_bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
  2990. &first_bad, &bad_sectors);
  2991. if (s->blocked_rdev == NULL
  2992. && (test_bit(Blocked, &rdev->flags)
  2993. || is_bad < 0)) {
  2994. if (is_bad < 0)
  2995. set_bit(BlockedBadBlocks,
  2996. &rdev->flags);
  2997. s->blocked_rdev = rdev;
  2998. atomic_inc(&rdev->nr_pending);
  2999. }
  3000. }
  3001. clear_bit(R5_Insync, &dev->flags);
  3002. if (!rdev)
  3003. /* Not in-sync */;
  3004. else if (is_bad) {
  3005. /* also not in-sync */
  3006. if (!test_bit(WriteErrorSeen, &rdev->flags) &&
  3007. test_bit(R5_UPTODATE, &dev->flags)) {
  3008. /* treat as in-sync, but with a read error
  3009. * which we can now try to correct
  3010. */
  3011. set_bit(R5_Insync, &dev->flags);
  3012. set_bit(R5_ReadError, &dev->flags);
  3013. }
  3014. } else if (test_bit(In_sync, &rdev->flags))
  3015. set_bit(R5_Insync, &dev->flags);
  3016. else if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset)
  3017. /* in sync if before recovery_offset */
  3018. set_bit(R5_Insync, &dev->flags);
  3019. else if (test_bit(R5_UPTODATE, &dev->flags) &&
  3020. test_bit(R5_Expanded, &dev->flags))
  3021. /* If we've reshaped into here, we assume it is Insync.
  3022. * We will shortly update recovery_offset to make
  3023. * it official.
  3024. */
  3025. set_bit(R5_Insync, &dev->flags);
  3026. if (rdev && test_bit(R5_WriteError, &dev->flags)) {
  3027. /* This flag does not apply to '.replacement'
  3028. * only to .rdev, so make sure to check that*/
  3029. struct md_rdev *rdev2 = rcu_dereference(
  3030. conf->disks[i].rdev);
  3031. if (rdev2 == rdev)
  3032. clear_bit(R5_Insync, &dev->flags);
  3033. if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
  3034. s->handle_bad_blocks = 1;
  3035. atomic_inc(&rdev2->nr_pending);
  3036. } else
  3037. clear_bit(R5_WriteError, &dev->flags);
  3038. }
  3039. if (rdev && test_bit(R5_MadeGood, &dev->flags)) {
  3040. /* This flag does not apply to '.replacement'
  3041. * only to .rdev, so make sure to check that*/
  3042. struct md_rdev *rdev2 = rcu_dereference(
  3043. conf->disks[i].rdev);
  3044. if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
  3045. s->handle_bad_blocks = 1;
  3046. atomic_inc(&rdev2->nr_pending);
  3047. } else
  3048. clear_bit(R5_MadeGood, &dev->flags);
  3049. }
  3050. if (test_bit(R5_MadeGoodRepl, &dev->flags)) {
  3051. struct md_rdev *rdev2 = rcu_dereference(
  3052. conf->disks[i].replacement);
  3053. if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
  3054. s->handle_bad_blocks = 1;
  3055. atomic_inc(&rdev2->nr_pending);
  3056. } else
  3057. clear_bit(R5_MadeGoodRepl, &dev->flags);
  3058. }
  3059. if (!test_bit(R5_Insync, &dev->flags)) {
  3060. /* The ReadError flag will just be confusing now */
  3061. clear_bit(R5_ReadError, &dev->flags);
  3062. clear_bit(R5_ReWrite, &dev->flags);
  3063. }
  3064. if (test_bit(R5_ReadError, &dev->flags))
  3065. clear_bit(R5_Insync, &dev->flags);
  3066. if (!test_bit(R5_Insync, &dev->flags)) {
  3067. if (s->failed < 2)
  3068. s->failed_num[s->failed] = i;
  3069. s->failed++;
  3070. if (rdev && !test_bit(Faulty, &rdev->flags))
  3071. do_recovery = 1;
  3072. }
  3073. }
  3074. if (test_bit(STRIPE_SYNCING, &sh->state)) {
  3075. /* If there is a failed device being replaced,
  3076. * we must be recovering.
  3077. * else if we are after recovery_cp, we must be syncing
  3078. * else if MD_RECOVERY_REQUESTED is set, we also are syncing.
  3079. * else we can only be replacing
  3080. * sync and recovery both need to read all devices, and so
  3081. * use the same flag.
  3082. */
  3083. if (do_recovery ||
  3084. sh->sector >= conf->mddev->recovery_cp ||
  3085. test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery)))
  3086. s->syncing = 1;
  3087. else
  3088. s->replacing = 1;
  3089. }
  3090. rcu_read_unlock();
  3091. }
  3092. static void handle_stripe(struct stripe_head *sh)
  3093. {
  3094. struct stripe_head_state s;
  3095. struct r5conf *conf = sh->raid_conf;
  3096. int i;
  3097. int prexor;
  3098. int disks = sh->disks;
  3099. struct r5dev *pdev, *qdev;
  3100. clear_bit(STRIPE_HANDLE, &sh->state);
  3101. if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) {
  3102. /* already being handled, ensure it gets handled
  3103. * again when current action finishes */
  3104. set_bit(STRIPE_HANDLE, &sh->state);
  3105. return;
  3106. }
  3107. if (test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {
  3108. set_bit(STRIPE_SYNCING, &sh->state);
  3109. clear_bit(STRIPE_INSYNC, &sh->state);
  3110. }
  3111. clear_bit(STRIPE_DELAYED, &sh->state);
  3112. pr_debug("handling stripe %llu, state=%#lx cnt=%d, "
  3113. "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n",
  3114. (unsigned long long)sh->sector, sh->state,
  3115. atomic_read(&sh->count), sh->pd_idx, sh->qd_idx,
  3116. sh->check_state, sh->reconstruct_state);
  3117. analyse_stripe(sh, &s);
  3118. if (s.handle_bad_blocks) {
  3119. set_bit(STRIPE_HANDLE, &sh->state);
  3120. goto finish;
  3121. }
  3122. if (unlikely(s.blocked_rdev)) {
  3123. if (s.syncing || s.expanding || s.expanded ||
  3124. s.replacing || s.to_write || s.written) {
  3125. set_bit(STRIPE_HANDLE, &sh->state);
  3126. goto finish;
  3127. }
  3128. /* There is nothing for the blocked_rdev to block */
  3129. rdev_dec_pending(s.blocked_rdev, conf->mddev);
  3130. s.blocked_rdev = NULL;
  3131. }
  3132. if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {
  3133. set_bit(STRIPE_OP_BIOFILL, &s.ops_request);
  3134. set_bit(STRIPE_BIOFILL_RUN, &sh->state);
  3135. }
  3136. pr_debug("locked=%d uptodate=%d to_read=%d"
  3137. " to_write=%d failed=%d failed_num=%d,%d\n",
  3138. s.locked, s.uptodate, s.to_read, s.to_write, s.failed,
  3139. s.failed_num[0], s.failed_num[1]);
  3140. /* check if the array has lost more than max_degraded devices and,
  3141. * if so, some requests might need to be failed.
  3142. */
  3143. if (s.failed > conf->max_degraded) {
  3144. sh->check_state = 0;
  3145. sh->reconstruct_state = 0;
  3146. if (s.to_read+s.to_write+s.written)
  3147. handle_failed_stripe(conf, sh, &s, disks, &s.return_bi);
  3148. if (s.syncing + s.replacing)
  3149. handle_failed_sync(conf, sh, &s);
  3150. }
  3151. /* Now we check to see if any write operations have recently
  3152. * completed
  3153. */
  3154. prexor = 0;
  3155. if (sh->reconstruct_state == reconstruct_state_prexor_drain_result)
  3156. prexor = 1;
  3157. if (sh->reconstruct_state == reconstruct_state_drain_result ||
  3158. sh->reconstruct_state == reconstruct_state_prexor_drain_result) {
  3159. sh->reconstruct_state = reconstruct_state_idle;
  3160. /* All the 'written' buffers and the parity block are ready to
  3161. * be written back to disk
  3162. */
  3163. BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) &&
  3164. !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags));
  3165. BUG_ON(sh->qd_idx >= 0 &&
  3166. !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) &&
  3167. !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags));
  3168. for (i = disks; i--; ) {
  3169. struct r5dev *dev = &sh->dev[i];
  3170. if (test_bit(R5_LOCKED, &dev->flags) &&
  3171. (i == sh->pd_idx || i == sh->qd_idx ||
  3172. dev->written)) {
  3173. pr_debug("Writing block %d\n", i);
  3174. set_bit(R5_Wantwrite, &dev->flags);
  3175. if (prexor)
  3176. continue;
  3177. if (!test_bit(R5_Insync, &dev->flags) ||
  3178. ((i == sh->pd_idx || i == sh->qd_idx) &&
  3179. s.failed == 0))
  3180. set_bit(STRIPE_INSYNC, &sh->state);
  3181. }
  3182. }
  3183. if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  3184. s.dec_preread_active = 1;
  3185. }
  3186. /*
  3187. * might be able to return some write requests if the parity blocks
  3188. * are safe, or on a failed drive
  3189. */
  3190. pdev = &sh->dev[sh->pd_idx];
  3191. s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx)
  3192. || (s.failed >= 2 && s.failed_num[1] == sh->pd_idx);
  3193. qdev = &sh->dev[sh->qd_idx];
  3194. s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx)
  3195. || (s.failed >= 2 && s.failed_num[1] == sh->qd_idx)
  3196. || conf->level < 6;
  3197. if (s.written &&
  3198. (s.p_failed || ((test_bit(R5_Insync, &pdev->flags)
  3199. && !test_bit(R5_LOCKED, &pdev->flags)
  3200. && (test_bit(R5_UPTODATE, &pdev->flags) ||
  3201. test_bit(R5_Discard, &pdev->flags))))) &&
  3202. (s.q_failed || ((test_bit(R5_Insync, &qdev->flags)
  3203. && !test_bit(R5_LOCKED, &qdev->flags)
  3204. && (test_bit(R5_UPTODATE, &qdev->flags) ||
  3205. test_bit(R5_Discard, &qdev->flags))))))
  3206. handle_stripe_clean_event(conf, sh, disks, &s.return_bi);
  3207. /* Now we might consider reading some blocks, either to check/generate
  3208. * parity, or to satisfy requests
  3209. * or to load a block that is being partially written.
  3210. */
  3211. if (s.to_read || s.non_overwrite
  3212. || (conf->level == 6 && s.to_write && s.failed)
  3213. || (s.syncing && (s.uptodate + s.compute < disks))
  3214. || s.replacing
  3215. || s.expanding)
  3216. handle_stripe_fill(sh, &s, disks);
  3217. /* Now to consider new write requests and what else, if anything
  3218. * should be read. We do not handle new writes when:
  3219. * 1/ A 'write' operation (copy+xor) is already in flight.
  3220. * 2/ A 'check' operation is in flight, as it may clobber the parity
  3221. * block.
  3222. */
  3223. if (s.to_write && !sh->reconstruct_state && !sh->check_state)
  3224. handle_stripe_dirtying(conf, sh, &s, disks);
  3225. /* maybe we need to check and possibly fix the parity for this stripe
  3226. * Any reads will already have been scheduled, so we just see if enough
  3227. * data is available. The parity check is held off while parity
  3228. * dependent operations are in flight.
  3229. */
  3230. if (sh->check_state ||
  3231. (s.syncing && s.locked == 0 &&
  3232. !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
  3233. !test_bit(STRIPE_INSYNC, &sh->state))) {
  3234. if (conf->level == 6)
  3235. handle_parity_checks6(conf, sh, &s, disks);
  3236. else
  3237. handle_parity_checks5(conf, sh, &s, disks);
  3238. }
  3239. if (s.replacing && s.locked == 0
  3240. && !test_bit(STRIPE_INSYNC, &sh->state)) {
  3241. /* Write out to replacement devices where possible */
  3242. for (i = 0; i < conf->raid_disks; i++)
  3243. if (test_bit(R5_UPTODATE, &sh->dev[i].flags) &&
  3244. test_bit(R5_NeedReplace, &sh->dev[i].flags)) {
  3245. set_bit(R5_WantReplace, &sh->dev[i].flags);
  3246. set_bit(R5_LOCKED, &sh->dev[i].flags);
  3247. s.locked++;
  3248. }
  3249. set_bit(STRIPE_INSYNC, &sh->state);
  3250. }
  3251. if ((s.syncing || s.replacing) && s.locked == 0 &&
  3252. test_bit(STRIPE_INSYNC, &sh->state)) {
  3253. md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
  3254. clear_bit(STRIPE_SYNCING, &sh->state);
  3255. }
  3256. /* If the failed drives are just a ReadError, then we might need
  3257. * to progress the repair/check process
  3258. */
  3259. if (s.failed <= conf->max_degraded && !conf->mddev->ro)
  3260. for (i = 0; i < s.failed; i++) {
  3261. struct r5dev *dev = &sh->dev[s.failed_num[i]];
  3262. if (test_bit(R5_ReadError, &dev->flags)
  3263. && !test_bit(R5_LOCKED, &dev->flags)
  3264. && test_bit(R5_UPTODATE, &dev->flags)
  3265. ) {
  3266. if (!test_bit(R5_ReWrite, &dev->flags)) {
  3267. set_bit(R5_Wantwrite, &dev->flags);
  3268. set_bit(R5_ReWrite, &dev->flags);
  3269. set_bit(R5_LOCKED, &dev->flags);
  3270. s.locked++;
  3271. } else {
  3272. /* let's read it back */
  3273. set_bit(R5_Wantread, &dev->flags);
  3274. set_bit(R5_LOCKED, &dev->flags);
  3275. s.locked++;
  3276. }
  3277. }
  3278. }
  3279. /* Finish reconstruct operations initiated by the expansion process */
  3280. if (sh->reconstruct_state == reconstruct_state_result) {
  3281. struct stripe_head *sh_src
  3282. = get_active_stripe(conf, sh->sector, 1, 1, 1);
  3283. if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) {
  3284. /* sh cannot be written until sh_src has been read.
  3285. * so arrange for sh to be delayed a little
  3286. */
  3287. set_bit(STRIPE_DELAYED, &sh->state);
  3288. set_bit(STRIPE_HANDLE, &sh->state);
  3289. if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE,
  3290. &sh_src->state))
  3291. atomic_inc(&conf->preread_active_stripes);
  3292. release_stripe(sh_src);
  3293. goto finish;
  3294. }
  3295. if (sh_src)
  3296. release_stripe(sh_src);
  3297. sh->reconstruct_state = reconstruct_state_idle;
  3298. clear_bit(STRIPE_EXPANDING, &sh->state);
  3299. for (i = conf->raid_disks; i--; ) {
  3300. set_bit(R5_Wantwrite, &sh->dev[i].flags);
  3301. set_bit(R5_LOCKED, &sh->dev[i].flags);
  3302. s.locked++;
  3303. }
  3304. }
  3305. if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&
  3306. !sh->reconstruct_state) {
  3307. /* Need to write out all blocks after computing parity */
  3308. sh->disks = conf->raid_disks;
  3309. stripe_set_idx(sh->sector, conf, 0, sh);
  3310. schedule_reconstruction(sh, &s, 1, 1);
  3311. } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {
  3312. clear_bit(STRIPE_EXPAND_READY, &sh->state);
  3313. atomic_dec(&conf->reshape_stripes);
  3314. wake_up(&conf->wait_for_overlap);
  3315. md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
  3316. }
  3317. if (s.expanding && s.locked == 0 &&
  3318. !test_bit(STRIPE_COMPUTE_RUN, &sh->state))
  3319. handle_stripe_expansion(conf, sh);
  3320. finish:
  3321. /* wait for this device to become unblocked */
  3322. if (unlikely(s.blocked_rdev)) {
  3323. if (conf->mddev->external)
  3324. md_wait_for_blocked_rdev(s.blocked_rdev,
  3325. conf->mddev);
  3326. else
  3327. /* Internal metadata will immediately
  3328. * be written by raid5d, so we don't
  3329. * need to wait here.
  3330. */
  3331. rdev_dec_pending(s.blocked_rdev,
  3332. conf->mddev);
  3333. }
  3334. if (s.handle_bad_blocks)
  3335. for (i = disks; i--; ) {
  3336. struct md_rdev *rdev;
  3337. struct r5dev *dev = &sh->dev[i];
  3338. if (test_and_clear_bit(R5_WriteError, &dev->flags)) {
  3339. /* We own a safe reference to the rdev */
  3340. rdev = conf->disks[i].rdev;
  3341. if (!rdev_set_badblocks(rdev, sh->sector,
  3342. STRIPE_SECTORS, 0))
  3343. md_error(conf->mddev, rdev);
  3344. rdev_dec_pending(rdev, conf->mddev);
  3345. }
  3346. if (test_and_clear_bit(R5_MadeGood, &dev->flags)) {
  3347. rdev = conf->disks[i].rdev;
  3348. rdev_clear_badblocks(rdev, sh->sector,
  3349. STRIPE_SECTORS, 0);
  3350. rdev_dec_pending(rdev, conf->mddev);
  3351. }
  3352. if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) {
  3353. rdev = conf->disks[i].replacement;
  3354. if (!rdev)
  3355. /* rdev have been moved down */
  3356. rdev = conf->disks[i].rdev;
  3357. rdev_clear_badblocks(rdev, sh->sector,
  3358. STRIPE_SECTORS, 0);
  3359. rdev_dec_pending(rdev, conf->mddev);
  3360. }
  3361. }
  3362. if (s.ops_request)
  3363. raid_run_ops(sh, s.ops_request);
  3364. ops_run_io(sh, &s);
  3365. if (s.dec_preread_active) {
  3366. /* We delay this until after ops_run_io so that if make_request
  3367. * is waiting on a flush, it won't continue until the writes
  3368. * have actually been submitted.
  3369. */
  3370. atomic_dec(&conf->preread_active_stripes);
  3371. if (atomic_read(&conf->preread_active_stripes) <
  3372. IO_THRESHOLD)
  3373. md_wakeup_thread(conf->mddev->thread);
  3374. }
  3375. return_io(s.return_bi);
  3376. clear_bit_unlock(STRIPE_ACTIVE, &sh->state);
  3377. }
  3378. static void raid5_activate_delayed(struct r5conf *conf)
  3379. {
  3380. if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) {
  3381. while (!list_empty(&conf->delayed_list)) {
  3382. struct list_head *l = conf->delayed_list.next;
  3383. struct stripe_head *sh;
  3384. sh = list_entry(l, struct stripe_head, lru);
  3385. list_del_init(l);
  3386. clear_bit(STRIPE_DELAYED, &sh->state);
  3387. if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  3388. atomic_inc(&conf->preread_active_stripes);
  3389. list_add_tail(&sh->lru, &conf->hold_list);
  3390. }
  3391. }
  3392. }
  3393. static void activate_bit_delay(struct r5conf *conf)
  3394. {
  3395. /* device_lock is held */
  3396. struct list_head head;
  3397. list_add(&head, &conf->bitmap_list);
  3398. list_del_init(&conf->bitmap_list);
  3399. while (!list_empty(&head)) {
  3400. struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru);
  3401. list_del_init(&sh->lru);
  3402. atomic_inc(&sh->count);
  3403. __release_stripe(conf, sh);
  3404. }
  3405. }
  3406. int md_raid5_congested(struct mddev *mddev, int bits)
  3407. {
  3408. struct r5conf *conf = mddev->private;
  3409. /* No difference between reads and writes. Just check
  3410. * how busy the stripe_cache is
  3411. */
  3412. if (conf->inactive_blocked)
  3413. return 1;
  3414. if (conf->quiesce)
  3415. return 1;
  3416. if (list_empty_careful(&conf->inactive_list))
  3417. return 1;
  3418. return 0;
  3419. }
  3420. EXPORT_SYMBOL_GPL(md_raid5_congested);
  3421. static int raid5_congested(void *data, int bits)
  3422. {
  3423. struct mddev *mddev = data;
  3424. return mddev_congested(mddev, bits) ||
  3425. md_raid5_congested(mddev, bits);
  3426. }
  3427. /* We want read requests to align with chunks where possible,
  3428. * but write requests don't need to.
  3429. */
  3430. static int raid5_mergeable_bvec(struct request_queue *q,
  3431. struct bvec_merge_data *bvm,
  3432. struct bio_vec *biovec)
  3433. {
  3434. struct mddev *mddev = q->queuedata;
  3435. sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
  3436. int max;
  3437. unsigned int chunk_sectors = mddev->chunk_sectors;
  3438. unsigned int bio_sectors = bvm->bi_size >> 9;
  3439. if ((bvm->bi_rw & 1) == WRITE)
  3440. return biovec->bv_len; /* always allow writes to be mergeable */
  3441. if (mddev->new_chunk_sectors < mddev->chunk_sectors)
  3442. chunk_sectors = mddev->new_chunk_sectors;
  3443. max = (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9;
  3444. if (max < 0) max = 0;
  3445. if (max <= biovec->bv_len && bio_sectors == 0)
  3446. return biovec->bv_len;
  3447. else
  3448. return max;
  3449. }
  3450. static int in_chunk_boundary(struct mddev *mddev, struct bio *bio)
  3451. {
  3452. sector_t sector = bio->bi_sector + get_start_sect(bio->bi_bdev);
  3453. unsigned int chunk_sectors = mddev->chunk_sectors;
  3454. unsigned int bio_sectors = bio->bi_size >> 9;
  3455. if (mddev->new_chunk_sectors < mddev->chunk_sectors)
  3456. chunk_sectors = mddev->new_chunk_sectors;
  3457. return chunk_sectors >=
  3458. ((sector & (chunk_sectors - 1)) + bio_sectors);
  3459. }
  3460. /*
  3461. * add bio to the retry LIFO ( in O(1) ... we are in interrupt )
  3462. * later sampled by raid5d.
  3463. */
  3464. static void add_bio_to_retry(struct bio *bi,struct r5conf *conf)
  3465. {
  3466. unsigned long flags;
  3467. spin_lock_irqsave(&conf->device_lock, flags);
  3468. bi->bi_next = conf->retry_read_aligned_list;
  3469. conf->retry_read_aligned_list = bi;
  3470. spin_unlock_irqrestore(&conf->device_lock, flags);
  3471. md_wakeup_thread(conf->mddev->thread);
  3472. }
  3473. static struct bio *remove_bio_from_retry(struct r5conf *conf)
  3474. {
  3475. struct bio *bi;
  3476. bi = conf->retry_read_aligned;
  3477. if (bi) {
  3478. conf->retry_read_aligned = NULL;
  3479. return bi;
  3480. }
  3481. bi = conf->retry_read_aligned_list;
  3482. if(bi) {
  3483. conf->retry_read_aligned_list = bi->bi_next;
  3484. bi->bi_next = NULL;
  3485. /*
  3486. * this sets the active strip count to 1 and the processed
  3487. * strip count to zero (upper 8 bits)
  3488. */
  3489. raid5_set_bi_stripes(bi, 1); /* biased count of active stripes */
  3490. }
  3491. return bi;
  3492. }
  3493. /*
  3494. * The "raid5_align_endio" should check if the read succeeded and if it
  3495. * did, call bio_endio on the original bio (having bio_put the new bio
  3496. * first).
  3497. * If the read failed..
  3498. */
  3499. static void raid5_align_endio(struct bio *bi, int error)
  3500. {
  3501. struct bio* raid_bi = bi->bi_private;
  3502. struct mddev *mddev;
  3503. struct r5conf *conf;
  3504. int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
  3505. struct md_rdev *rdev;
  3506. bio_put(bi);
  3507. rdev = (void*)raid_bi->bi_next;
  3508. raid_bi->bi_next = NULL;
  3509. mddev = rdev->mddev;
  3510. conf = mddev->private;
  3511. rdev_dec_pending(rdev, conf->mddev);
  3512. if (!error && uptodate) {
  3513. bio_endio(raid_bi, 0);
  3514. if (atomic_dec_and_test(&conf->active_aligned_reads))
  3515. wake_up(&conf->wait_for_stripe);
  3516. return;
  3517. }
  3518. pr_debug("raid5_align_endio : io error...handing IO for a retry\n");
  3519. add_bio_to_retry(raid_bi, conf);
  3520. }
  3521. static int bio_fits_rdev(struct bio *bi)
  3522. {
  3523. struct request_queue *q = bdev_get_queue(bi->bi_bdev);
  3524. if ((bi->bi_size>>9) > queue_max_sectors(q))
  3525. return 0;
  3526. blk_recount_segments(q, bi);
  3527. if (bi->bi_phys_segments > queue_max_segments(q))
  3528. return 0;
  3529. if (q->merge_bvec_fn)
  3530. /* it's too hard to apply the merge_bvec_fn at this stage,
  3531. * just just give up
  3532. */
  3533. return 0;
  3534. return 1;
  3535. }
  3536. static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio)
  3537. {
  3538. struct r5conf *conf = mddev->private;
  3539. int dd_idx;
  3540. struct bio* align_bi;
  3541. struct md_rdev *rdev;
  3542. sector_t end_sector;
  3543. if (!in_chunk_boundary(mddev, raid_bio)) {
  3544. pr_debug("chunk_aligned_read : non aligned\n");
  3545. return 0;
  3546. }
  3547. /*
  3548. * use bio_clone_mddev to make a copy of the bio
  3549. */
  3550. align_bi = bio_clone_mddev(raid_bio, GFP_NOIO, mddev);
  3551. if (!align_bi)
  3552. return 0;
  3553. /*
  3554. * set bi_end_io to a new function, and set bi_private to the
  3555. * original bio.
  3556. */
  3557. align_bi->bi_end_io = raid5_align_endio;
  3558. align_bi->bi_private = raid_bio;
  3559. /*
  3560. * compute position
  3561. */
  3562. align_bi->bi_sector = raid5_compute_sector(conf, raid_bio->bi_sector,
  3563. 0,
  3564. &dd_idx, NULL);
  3565. end_sector = align_bi->bi_sector + (align_bi->bi_size >> 9);
  3566. rcu_read_lock();
  3567. rdev = rcu_dereference(conf->disks[dd_idx].replacement);
  3568. if (!rdev || test_bit(Faulty, &rdev->flags) ||
  3569. rdev->recovery_offset < end_sector) {
  3570. rdev = rcu_dereference(conf->disks[dd_idx].rdev);
  3571. if (rdev &&
  3572. (test_bit(Faulty, &rdev->flags) ||
  3573. !(test_bit(In_sync, &rdev->flags) ||
  3574. rdev->recovery_offset >= end_sector)))
  3575. rdev = NULL;
  3576. }
  3577. if (rdev) {
  3578. sector_t first_bad;
  3579. int bad_sectors;
  3580. atomic_inc(&rdev->nr_pending);
  3581. rcu_read_unlock();
  3582. raid_bio->bi_next = (void*)rdev;
  3583. align_bi->bi_bdev = rdev->bdev;
  3584. align_bi->bi_flags &= ~(1 << BIO_SEG_VALID);
  3585. if (!bio_fits_rdev(align_bi) ||
  3586. is_badblock(rdev, align_bi->bi_sector, align_bi->bi_size>>9,
  3587. &first_bad, &bad_sectors)) {
  3588. /* too big in some way, or has a known bad block */
  3589. bio_put(align_bi);
  3590. rdev_dec_pending(rdev, mddev);
  3591. return 0;
  3592. }
  3593. /* No reshape active, so we can trust rdev->data_offset */
  3594. align_bi->bi_sector += rdev->data_offset;
  3595. spin_lock_irq(&conf->device_lock);
  3596. wait_event_lock_irq(conf->wait_for_stripe,
  3597. conf->quiesce == 0,
  3598. conf->device_lock);
  3599. atomic_inc(&conf->active_aligned_reads);
  3600. spin_unlock_irq(&conf->device_lock);
  3601. trace_block_bio_remap(bdev_get_queue(align_bi->bi_bdev),
  3602. align_bi, disk_devt(mddev->gendisk),
  3603. raid_bio->bi_sector);
  3604. generic_make_request(align_bi);
  3605. return 1;
  3606. } else {
  3607. rcu_read_unlock();
  3608. bio_put(align_bi);
  3609. return 0;
  3610. }
  3611. }
  3612. /* __get_priority_stripe - get the next stripe to process
  3613. *
  3614. * Full stripe writes are allowed to pass preread active stripes up until
  3615. * the bypass_threshold is exceeded. In general the bypass_count
  3616. * increments when the handle_list is handled before the hold_list; however, it
  3617. * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a
  3618. * stripe with in flight i/o. The bypass_count will be reset when the
  3619. * head of the hold_list has changed, i.e. the head was promoted to the
  3620. * handle_list.
  3621. */
  3622. static struct stripe_head *__get_priority_stripe(struct r5conf *conf)
  3623. {
  3624. struct stripe_head *sh;
  3625. pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n",
  3626. __func__,
  3627. list_empty(&conf->handle_list) ? "empty" : "busy",
  3628. list_empty(&conf->hold_list) ? "empty" : "busy",
  3629. atomic_read(&conf->pending_full_writes), conf->bypass_count);
  3630. if (!list_empty(&conf->handle_list)) {
  3631. sh = list_entry(conf->handle_list.next, typeof(*sh), lru);
  3632. if (list_empty(&conf->hold_list))
  3633. conf->bypass_count = 0;
  3634. else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) {
  3635. if (conf->hold_list.next == conf->last_hold)
  3636. conf->bypass_count++;
  3637. else {
  3638. conf->last_hold = conf->hold_list.next;
  3639. conf->bypass_count -= conf->bypass_threshold;
  3640. if (conf->bypass_count < 0)
  3641. conf->bypass_count = 0;
  3642. }
  3643. }
  3644. } else if (!list_empty(&conf->hold_list) &&
  3645. ((conf->bypass_threshold &&
  3646. conf->bypass_count > conf->bypass_threshold) ||
  3647. atomic_read(&conf->pending_full_writes) == 0)) {
  3648. sh = list_entry(conf->hold_list.next,
  3649. typeof(*sh), lru);
  3650. conf->bypass_count -= conf->bypass_threshold;
  3651. if (conf->bypass_count < 0)
  3652. conf->bypass_count = 0;
  3653. } else
  3654. return NULL;
  3655. list_del_init(&sh->lru);
  3656. atomic_inc(&sh->count);
  3657. BUG_ON(atomic_read(&sh->count) != 1);
  3658. return sh;
  3659. }
  3660. struct raid5_plug_cb {
  3661. struct blk_plug_cb cb;
  3662. struct list_head list;
  3663. };
  3664. static void raid5_unplug(struct blk_plug_cb *blk_cb, bool from_schedule)
  3665. {
  3666. struct raid5_plug_cb *cb = container_of(
  3667. blk_cb, struct raid5_plug_cb, cb);
  3668. struct stripe_head *sh;
  3669. struct mddev *mddev = cb->cb.data;
  3670. struct r5conf *conf = mddev->private;
  3671. int cnt = 0;
  3672. if (cb->list.next && !list_empty(&cb->list)) {
  3673. spin_lock_irq(&conf->device_lock);
  3674. while (!list_empty(&cb->list)) {
  3675. sh = list_first_entry(&cb->list, struct stripe_head, lru);
  3676. list_del_init(&sh->lru);
  3677. /*
  3678. * avoid race release_stripe_plug() sees
  3679. * STRIPE_ON_UNPLUG_LIST clear but the stripe
  3680. * is still in our list
  3681. */
  3682. smp_mb__before_clear_bit();
  3683. clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state);
  3684. __release_stripe(conf, sh);
  3685. cnt++;
  3686. }
  3687. spin_unlock_irq(&conf->device_lock);
  3688. }
  3689. trace_block_unplug(mddev->queue, cnt, !from_schedule);
  3690. kfree(cb);
  3691. }
  3692. static void release_stripe_plug(struct mddev *mddev,
  3693. struct stripe_head *sh)
  3694. {
  3695. struct blk_plug_cb *blk_cb = blk_check_plugged(
  3696. raid5_unplug, mddev,
  3697. sizeof(struct raid5_plug_cb));
  3698. struct raid5_plug_cb *cb;
  3699. if (!blk_cb) {
  3700. release_stripe(sh);
  3701. return;
  3702. }
  3703. cb = container_of(blk_cb, struct raid5_plug_cb, cb);
  3704. if (cb->list.next == NULL)
  3705. INIT_LIST_HEAD(&cb->list);
  3706. if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state))
  3707. list_add_tail(&sh->lru, &cb->list);
  3708. else
  3709. release_stripe(sh);
  3710. }
  3711. static void make_discard_request(struct mddev *mddev, struct bio *bi)
  3712. {
  3713. struct r5conf *conf = mddev->private;
  3714. sector_t logical_sector, last_sector;
  3715. struct stripe_head *sh;
  3716. int remaining;
  3717. int stripe_sectors;
  3718. if (mddev->reshape_position != MaxSector)
  3719. /* Skip discard while reshape is happening */
  3720. return;
  3721. logical_sector = bi->bi_sector & ~((sector_t)STRIPE_SECTORS-1);
  3722. last_sector = bi->bi_sector + (bi->bi_size>>9);
  3723. bi->bi_next = NULL;
  3724. bi->bi_phys_segments = 1; /* over-loaded to count active stripes */
  3725. stripe_sectors = conf->chunk_sectors *
  3726. (conf->raid_disks - conf->max_degraded);
  3727. logical_sector = DIV_ROUND_UP_SECTOR_T(logical_sector,
  3728. stripe_sectors);
  3729. sector_div(last_sector, stripe_sectors);
  3730. logical_sector *= conf->chunk_sectors;
  3731. last_sector *= conf->chunk_sectors;
  3732. for (; logical_sector < last_sector;
  3733. logical_sector += STRIPE_SECTORS) {
  3734. DEFINE_WAIT(w);
  3735. int d;
  3736. again:
  3737. sh = get_active_stripe(conf, logical_sector, 0, 0, 0);
  3738. prepare_to_wait(&conf->wait_for_overlap, &w,
  3739. TASK_UNINTERRUPTIBLE);
  3740. spin_lock_irq(&sh->stripe_lock);
  3741. for (d = 0; d < conf->raid_disks; d++) {
  3742. if (d == sh->pd_idx || d == sh->qd_idx)
  3743. continue;
  3744. if (sh->dev[d].towrite || sh->dev[d].toread) {
  3745. set_bit(R5_Overlap, &sh->dev[d].flags);
  3746. spin_unlock_irq(&sh->stripe_lock);
  3747. release_stripe(sh);
  3748. schedule();
  3749. goto again;
  3750. }
  3751. }
  3752. finish_wait(&conf->wait_for_overlap, &w);
  3753. for (d = 0; d < conf->raid_disks; d++) {
  3754. if (d == sh->pd_idx || d == sh->qd_idx)
  3755. continue;
  3756. sh->dev[d].towrite = bi;
  3757. set_bit(R5_OVERWRITE, &sh->dev[d].flags);
  3758. raid5_inc_bi_active_stripes(bi);
  3759. }
  3760. spin_unlock_irq(&sh->stripe_lock);
  3761. if (conf->mddev->bitmap) {
  3762. for (d = 0;
  3763. d < conf->raid_disks - conf->max_degraded;
  3764. d++)
  3765. bitmap_startwrite(mddev->bitmap,
  3766. sh->sector,
  3767. STRIPE_SECTORS,
  3768. 0);
  3769. sh->bm_seq = conf->seq_flush + 1;
  3770. set_bit(STRIPE_BIT_DELAY, &sh->state);
  3771. }
  3772. set_bit(STRIPE_HANDLE, &sh->state);
  3773. clear_bit(STRIPE_DELAYED, &sh->state);
  3774. if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  3775. atomic_inc(&conf->preread_active_stripes);
  3776. release_stripe_plug(mddev, sh);
  3777. }
  3778. remaining = raid5_dec_bi_active_stripes(bi);
  3779. if (remaining == 0) {
  3780. md_write_end(mddev);
  3781. bio_endio(bi, 0);
  3782. }
  3783. }
  3784. static void make_request(struct mddev *mddev, struct bio * bi)
  3785. {
  3786. struct r5conf *conf = mddev->private;
  3787. int dd_idx;
  3788. sector_t new_sector;
  3789. sector_t logical_sector, last_sector;
  3790. struct stripe_head *sh;
  3791. const int rw = bio_data_dir(bi);
  3792. int remaining;
  3793. if (unlikely(bi->bi_rw & REQ_FLUSH)) {
  3794. md_flush_request(mddev, bi);
  3795. return;
  3796. }
  3797. md_write_start(mddev, bi);
  3798. if (rw == READ &&
  3799. mddev->reshape_position == MaxSector &&
  3800. chunk_aligned_read(mddev,bi))
  3801. return;
  3802. if (unlikely(bi->bi_rw & REQ_DISCARD)) {
  3803. make_discard_request(mddev, bi);
  3804. return;
  3805. }
  3806. logical_sector = bi->bi_sector & ~((sector_t)STRIPE_SECTORS-1);
  3807. last_sector = bi->bi_sector + (bi->bi_size>>9);
  3808. bi->bi_next = NULL;
  3809. bi->bi_phys_segments = 1; /* over-loaded to count active stripes */
  3810. for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) {
  3811. DEFINE_WAIT(w);
  3812. int previous;
  3813. retry:
  3814. previous = 0;
  3815. prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE);
  3816. if (unlikely(conf->reshape_progress != MaxSector)) {
  3817. /* spinlock is needed as reshape_progress may be
  3818. * 64bit on a 32bit platform, and so it might be
  3819. * possible to see a half-updated value
  3820. * Of course reshape_progress could change after
  3821. * the lock is dropped, so once we get a reference
  3822. * to the stripe that we think it is, we will have
  3823. * to check again.
  3824. */
  3825. spin_lock_irq(&conf->device_lock);
  3826. if (mddev->reshape_backwards
  3827. ? logical_sector < conf->reshape_progress
  3828. : logical_sector >= conf->reshape_progress) {
  3829. previous = 1;
  3830. } else {
  3831. if (mddev->reshape_backwards
  3832. ? logical_sector < conf->reshape_safe
  3833. : logical_sector >= conf->reshape_safe) {
  3834. spin_unlock_irq(&conf->device_lock);
  3835. schedule();
  3836. goto retry;
  3837. }
  3838. }
  3839. spin_unlock_irq(&conf->device_lock);
  3840. }
  3841. new_sector = raid5_compute_sector(conf, logical_sector,
  3842. previous,
  3843. &dd_idx, NULL);
  3844. pr_debug("raid456: make_request, sector %llu logical %llu\n",
  3845. (unsigned long long)new_sector,
  3846. (unsigned long long)logical_sector);
  3847. sh = get_active_stripe(conf, new_sector, previous,
  3848. (bi->bi_rw&RWA_MASK), 0);
  3849. if (sh) {
  3850. if (unlikely(previous)) {
  3851. /* expansion might have moved on while waiting for a
  3852. * stripe, so we must do the range check again.
  3853. * Expansion could still move past after this
  3854. * test, but as we are holding a reference to
  3855. * 'sh', we know that if that happens,
  3856. * STRIPE_EXPANDING will get set and the expansion
  3857. * won't proceed until we finish with the stripe.
  3858. */
  3859. int must_retry = 0;
  3860. spin_lock_irq(&conf->device_lock);
  3861. if (mddev->reshape_backwards
  3862. ? logical_sector >= conf->reshape_progress
  3863. : logical_sector < conf->reshape_progress)
  3864. /* mismatch, need to try again */
  3865. must_retry = 1;
  3866. spin_unlock_irq(&conf->device_lock);
  3867. if (must_retry) {
  3868. release_stripe(sh);
  3869. schedule();
  3870. goto retry;
  3871. }
  3872. }
  3873. if (rw == WRITE &&
  3874. logical_sector >= mddev->suspend_lo &&
  3875. logical_sector < mddev->suspend_hi) {
  3876. release_stripe(sh);
  3877. /* As the suspend_* range is controlled by
  3878. * userspace, we want an interruptible
  3879. * wait.
  3880. */
  3881. flush_signals(current);
  3882. prepare_to_wait(&conf->wait_for_overlap,
  3883. &w, TASK_INTERRUPTIBLE);
  3884. if (logical_sector >= mddev->suspend_lo &&
  3885. logical_sector < mddev->suspend_hi)
  3886. schedule();
  3887. goto retry;
  3888. }
  3889. if (test_bit(STRIPE_EXPANDING, &sh->state) ||
  3890. !add_stripe_bio(sh, bi, dd_idx, rw)) {
  3891. /* Stripe is busy expanding or
  3892. * add failed due to overlap. Flush everything
  3893. * and wait a while
  3894. */
  3895. md_wakeup_thread(mddev->thread);
  3896. release_stripe(sh);
  3897. schedule();
  3898. goto retry;
  3899. }
  3900. finish_wait(&conf->wait_for_overlap, &w);
  3901. set_bit(STRIPE_HANDLE, &sh->state);
  3902. clear_bit(STRIPE_DELAYED, &sh->state);
  3903. if ((bi->bi_rw & REQ_SYNC) &&
  3904. !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  3905. atomic_inc(&conf->preread_active_stripes);
  3906. release_stripe_plug(mddev, sh);
  3907. } else {
  3908. /* cannot get stripe for read-ahead, just give-up */
  3909. clear_bit(BIO_UPTODATE, &bi->bi_flags);
  3910. finish_wait(&conf->wait_for_overlap, &w);
  3911. break;
  3912. }
  3913. }
  3914. remaining = raid5_dec_bi_active_stripes(bi);
  3915. if (remaining == 0) {
  3916. if ( rw == WRITE )
  3917. md_write_end(mddev);
  3918. bio_endio(bi, 0);
  3919. }
  3920. }
  3921. static sector_t raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks);
  3922. static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skipped)
  3923. {
  3924. /* reshaping is quite different to recovery/resync so it is
  3925. * handled quite separately ... here.
  3926. *
  3927. * On each call to sync_request, we gather one chunk worth of
  3928. * destination stripes and flag them as expanding.
  3929. * Then we find all the source stripes and request reads.
  3930. * As the reads complete, handle_stripe will copy the data
  3931. * into the destination stripe and release that stripe.
  3932. */
  3933. struct r5conf *conf = mddev->private;
  3934. struct stripe_head *sh;
  3935. sector_t first_sector, last_sector;
  3936. int raid_disks = conf->previous_raid_disks;
  3937. int data_disks = raid_disks - conf->max_degraded;
  3938. int new_data_disks = conf->raid_disks - conf->max_degraded;
  3939. int i;
  3940. int dd_idx;
  3941. sector_t writepos, readpos, safepos;
  3942. sector_t stripe_addr;
  3943. int reshape_sectors;
  3944. struct list_head stripes;
  3945. if (sector_nr == 0) {
  3946. /* If restarting in the middle, skip the initial sectors */
  3947. if (mddev->reshape_backwards &&
  3948. conf->reshape_progress < raid5_size(mddev, 0, 0)) {
  3949. sector_nr = raid5_size(mddev, 0, 0)
  3950. - conf->reshape_progress;
  3951. } else if (!mddev->reshape_backwards &&
  3952. conf->reshape_progress > 0)
  3953. sector_nr = conf->reshape_progress;
  3954. sector_div(sector_nr, new_data_disks);
  3955. if (sector_nr) {
  3956. mddev->curr_resync_completed = sector_nr;
  3957. sysfs_notify(&mddev->kobj, NULL, "sync_completed");
  3958. *skipped = 1;
  3959. return sector_nr;
  3960. }
  3961. }
  3962. /* We need to process a full chunk at a time.
  3963. * If old and new chunk sizes differ, we need to process the
  3964. * largest of these
  3965. */
  3966. if (mddev->new_chunk_sectors > mddev->chunk_sectors)
  3967. reshape_sectors = mddev->new_chunk_sectors;
  3968. else
  3969. reshape_sectors = mddev->chunk_sectors;
  3970. /* We update the metadata at least every 10 seconds, or when
  3971. * the data about to be copied would over-write the source of
  3972. * the data at the front of the range. i.e. one new_stripe
  3973. * along from reshape_progress new_maps to after where
  3974. * reshape_safe old_maps to
  3975. */
  3976. writepos = conf->reshape_progress;
  3977. sector_div(writepos, new_data_disks);
  3978. readpos = conf->reshape_progress;
  3979. sector_div(readpos, data_disks);
  3980. safepos = conf->reshape_safe;
  3981. sector_div(safepos, data_disks);
  3982. if (mddev->reshape_backwards) {
  3983. writepos -= min_t(sector_t, reshape_sectors, writepos);
  3984. readpos += reshape_sectors;
  3985. safepos += reshape_sectors;
  3986. } else {
  3987. writepos += reshape_sectors;
  3988. readpos -= min_t(sector_t, reshape_sectors, readpos);
  3989. safepos -= min_t(sector_t, reshape_sectors, safepos);
  3990. }
  3991. /* Having calculated the 'writepos' possibly use it
  3992. * to set 'stripe_addr' which is where we will write to.
  3993. */
  3994. if (mddev->reshape_backwards) {
  3995. BUG_ON(conf->reshape_progress == 0);
  3996. stripe_addr = writepos;
  3997. BUG_ON((mddev->dev_sectors &
  3998. ~((sector_t)reshape_sectors - 1))
  3999. - reshape_sectors - stripe_addr
  4000. != sector_nr);
  4001. } else {
  4002. BUG_ON(writepos != sector_nr + reshape_sectors);
  4003. stripe_addr = sector_nr;
  4004. }
  4005. /* 'writepos' is the most advanced device address we might write.
  4006. * 'readpos' is the least advanced device address we might read.
  4007. * 'safepos' is the least address recorded in the metadata as having
  4008. * been reshaped.
  4009. * If there is a min_offset_diff, these are adjusted either by
  4010. * increasing the safepos/readpos if diff is negative, or
  4011. * increasing writepos if diff is positive.
  4012. * If 'readpos' is then behind 'writepos', there is no way that we can
  4013. * ensure safety in the face of a crash - that must be done by userspace
  4014. * making a backup of the data. So in that case there is no particular
  4015. * rush to update metadata.
  4016. * Otherwise if 'safepos' is behind 'writepos', then we really need to
  4017. * update the metadata to advance 'safepos' to match 'readpos' so that
  4018. * we can be safe in the event of a crash.
  4019. * So we insist on updating metadata if safepos is behind writepos and
  4020. * readpos is beyond writepos.
  4021. * In any case, update the metadata every 10 seconds.
  4022. * Maybe that number should be configurable, but I'm not sure it is
  4023. * worth it.... maybe it could be a multiple of safemode_delay???
  4024. */
  4025. if (conf->min_offset_diff < 0) {
  4026. safepos += -conf->min_offset_diff;
  4027. readpos += -conf->min_offset_diff;
  4028. } else
  4029. writepos += conf->min_offset_diff;
  4030. if ((mddev->reshape_backwards
  4031. ? (safepos > writepos && readpos < writepos)
  4032. : (safepos < writepos && readpos > writepos)) ||
  4033. time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
  4034. /* Cannot proceed until we've updated the superblock... */
  4035. wait_event(conf->wait_for_overlap,
  4036. atomic_read(&conf->reshape_stripes)==0);
  4037. mddev->reshape_position = conf->reshape_progress;
  4038. mddev->curr_resync_completed = sector_nr;
  4039. conf->reshape_checkpoint = jiffies;
  4040. set_bit(MD_CHANGE_DEVS, &mddev->flags);
  4041. md_wakeup_thread(mddev->thread);
  4042. wait_event(mddev->sb_wait, mddev->flags == 0 ||
  4043. kthread_should_stop());
  4044. spin_lock_irq(&conf->device_lock);
  4045. conf->reshape_safe = mddev->reshape_position;
  4046. spin_unlock_irq(&conf->device_lock);
  4047. wake_up(&conf->wait_for_overlap);
  4048. sysfs_notify(&mddev->kobj, NULL, "sync_completed");
  4049. }
  4050. INIT_LIST_HEAD(&stripes);
  4051. for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) {
  4052. int j;
  4053. int skipped_disk = 0;
  4054. sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1);
  4055. set_bit(STRIPE_EXPANDING, &sh->state);
  4056. atomic_inc(&conf->reshape_stripes);
  4057. /* If any of this stripe is beyond the end of the old
  4058. * array, then we need to zero those blocks
  4059. */
  4060. for (j=sh->disks; j--;) {
  4061. sector_t s;
  4062. if (j == sh->pd_idx)
  4063. continue;
  4064. if (conf->level == 6 &&
  4065. j == sh->qd_idx)
  4066. continue;
  4067. s = compute_blocknr(sh, j, 0);
  4068. if (s < raid5_size(mddev, 0, 0)) {
  4069. skipped_disk = 1;
  4070. continue;
  4071. }
  4072. memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE);
  4073. set_bit(R5_Expanded, &sh->dev[j].flags);
  4074. set_bit(R5_UPTODATE, &sh->dev[j].flags);
  4075. }
  4076. if (!skipped_disk) {
  4077. set_bit(STRIPE_EXPAND_READY, &sh->state);
  4078. set_bit(STRIPE_HANDLE, &sh->state);
  4079. }
  4080. list_add(&sh->lru, &stripes);
  4081. }
  4082. spin_lock_irq(&conf->device_lock);
  4083. if (mddev->reshape_backwards)
  4084. conf->reshape_progress -= reshape_sectors * new_data_disks;
  4085. else
  4086. conf->reshape_progress += reshape_sectors * new_data_disks;
  4087. spin_unlock_irq(&conf->device_lock);
  4088. /* Ok, those stripe are ready. We can start scheduling
  4089. * reads on the source stripes.
  4090. * The source stripes are determined by mapping the first and last
  4091. * block on the destination stripes.
  4092. */
  4093. first_sector =
  4094. raid5_compute_sector(conf, stripe_addr*(new_data_disks),
  4095. 1, &dd_idx, NULL);
  4096. last_sector =
  4097. raid5_compute_sector(conf, ((stripe_addr+reshape_sectors)
  4098. * new_data_disks - 1),
  4099. 1, &dd_idx, NULL);
  4100. if (last_sector >= mddev->dev_sectors)
  4101. last_sector = mddev->dev_sectors - 1;
  4102. while (first_sector <= last_sector) {
  4103. sh = get_active_stripe(conf, first_sector, 1, 0, 1);
  4104. set_bit(STRIPE_EXPAND_SOURCE, &sh->state);
  4105. set_bit(STRIPE_HANDLE, &sh->state);
  4106. release_stripe(sh);
  4107. first_sector += STRIPE_SECTORS;
  4108. }
  4109. /* Now that the sources are clearly marked, we can release
  4110. * the destination stripes
  4111. */
  4112. while (!list_empty(&stripes)) {
  4113. sh = list_entry(stripes.next, struct stripe_head, lru);
  4114. list_del_init(&sh->lru);
  4115. release_stripe(sh);
  4116. }
  4117. /* If this takes us to the resync_max point where we have to pause,
  4118. * then we need to write out the superblock.
  4119. */
  4120. sector_nr += reshape_sectors;
  4121. if ((sector_nr - mddev->curr_resync_completed) * 2
  4122. >= mddev->resync_max - mddev->curr_resync_completed) {
  4123. /* Cannot proceed until we've updated the superblock... */
  4124. wait_event(conf->wait_for_overlap,
  4125. atomic_read(&conf->reshape_stripes) == 0);
  4126. mddev->reshape_position = conf->reshape_progress;
  4127. mddev->curr_resync_completed = sector_nr;
  4128. conf->reshape_checkpoint = jiffies;
  4129. set_bit(MD_CHANGE_DEVS, &mddev->flags);
  4130. md_wakeup_thread(mddev->thread);
  4131. wait_event(mddev->sb_wait,
  4132. !test_bit(MD_CHANGE_DEVS, &mddev->flags)
  4133. || kthread_should_stop());
  4134. spin_lock_irq(&conf->device_lock);
  4135. conf->reshape_safe = mddev->reshape_position;
  4136. spin_unlock_irq(&conf->device_lock);
  4137. wake_up(&conf->wait_for_overlap);
  4138. sysfs_notify(&mddev->kobj, NULL, "sync_completed");
  4139. }
  4140. return reshape_sectors;
  4141. }
  4142. /* FIXME go_faster isn't used */
  4143. static inline sector_t sync_request(struct mddev *mddev, sector_t sector_nr, int *skipped, int go_faster)
  4144. {
  4145. struct r5conf *conf = mddev->private;
  4146. struct stripe_head *sh;
  4147. sector_t max_sector = mddev->dev_sectors;
  4148. sector_t sync_blocks;
  4149. int still_degraded = 0;
  4150. int i;
  4151. if (sector_nr >= max_sector) {
  4152. /* just being told to finish up .. nothing much to do */
  4153. if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
  4154. end_reshape(conf);
  4155. return 0;
  4156. }
  4157. if (mddev->curr_resync < max_sector) /* aborted */
  4158. bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
  4159. &sync_blocks, 1);
  4160. else /* completed sync */
  4161. conf->fullsync = 0;
  4162. bitmap_close_sync(mddev->bitmap);
  4163. return 0;
  4164. }
  4165. /* Allow raid5_quiesce to complete */
  4166. wait_event(conf->wait_for_overlap, conf->quiesce != 2);
  4167. if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
  4168. return reshape_request(mddev, sector_nr, skipped);
  4169. /* No need to check resync_max as we never do more than one
  4170. * stripe, and as resync_max will always be on a chunk boundary,
  4171. * if the check in md_do_sync didn't fire, there is no chance
  4172. * of overstepping resync_max here
  4173. */
  4174. /* if there is too many failed drives and we are trying
  4175. * to resync, then assert that we are finished, because there is
  4176. * nothing we can do.
  4177. */
  4178. if (mddev->degraded >= conf->max_degraded &&
  4179. test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
  4180. sector_t rv = mddev->dev_sectors - sector_nr;
  4181. *skipped = 1;
  4182. return rv;
  4183. }
  4184. if (!bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) &&
  4185. !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
  4186. !conf->fullsync && sync_blocks >= STRIPE_SECTORS) {
  4187. /* we can skip this block, and probably more */
  4188. sync_blocks /= STRIPE_SECTORS;
  4189. *skipped = 1;
  4190. return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */
  4191. }
  4192. bitmap_cond_end_sync(mddev->bitmap, sector_nr);
  4193. sh = get_active_stripe(conf, sector_nr, 0, 1, 0);
  4194. if (sh == NULL) {
  4195. sh = get_active_stripe(conf, sector_nr, 0, 0, 0);
  4196. /* make sure we don't swamp the stripe cache if someone else
  4197. * is trying to get access
  4198. */
  4199. schedule_timeout_uninterruptible(1);
  4200. }
  4201. /* Need to check if array will still be degraded after recovery/resync
  4202. * We don't need to check the 'failed' flag as when that gets set,
  4203. * recovery aborts.
  4204. */
  4205. for (i = 0; i < conf->raid_disks; i++)
  4206. if (conf->disks[i].rdev == NULL)
  4207. still_degraded = 1;
  4208. bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded);
  4209. set_bit(STRIPE_SYNC_REQUESTED, &sh->state);
  4210. handle_stripe(sh);
  4211. release_stripe(sh);
  4212. return STRIPE_SECTORS;
  4213. }
  4214. static int retry_aligned_read(struct r5conf *conf, struct bio *raid_bio)
  4215. {
  4216. /* We may not be able to submit a whole bio at once as there
  4217. * may not be enough stripe_heads available.
  4218. * We cannot pre-allocate enough stripe_heads as we may need
  4219. * more than exist in the cache (if we allow ever large chunks).
  4220. * So we do one stripe head at a time and record in
  4221. * ->bi_hw_segments how many have been done.
  4222. *
  4223. * We *know* that this entire raid_bio is in one chunk, so
  4224. * it will be only one 'dd_idx' and only need one call to raid5_compute_sector.
  4225. */
  4226. struct stripe_head *sh;
  4227. int dd_idx;
  4228. sector_t sector, logical_sector, last_sector;
  4229. int scnt = 0;
  4230. int remaining;
  4231. int handled = 0;
  4232. logical_sector = raid_bio->bi_sector & ~((sector_t)STRIPE_SECTORS-1);
  4233. sector = raid5_compute_sector(conf, logical_sector,
  4234. 0, &dd_idx, NULL);
  4235. last_sector = raid_bio->bi_sector + (raid_bio->bi_size>>9);
  4236. for (; logical_sector < last_sector;
  4237. logical_sector += STRIPE_SECTORS,
  4238. sector += STRIPE_SECTORS,
  4239. scnt++) {
  4240. if (scnt < raid5_bi_processed_stripes(raid_bio))
  4241. /* already done this stripe */
  4242. continue;
  4243. sh = get_active_stripe(conf, sector, 0, 1, 0);
  4244. if (!sh) {
  4245. /* failed to get a stripe - must wait */
  4246. raid5_set_bi_processed_stripes(raid_bio, scnt);
  4247. conf->retry_read_aligned = raid_bio;
  4248. return handled;
  4249. }
  4250. if (!add_stripe_bio(sh, raid_bio, dd_idx, 0)) {
  4251. release_stripe(sh);
  4252. raid5_set_bi_processed_stripes(raid_bio, scnt);
  4253. conf->retry_read_aligned = raid_bio;
  4254. return handled;
  4255. }
  4256. set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags);
  4257. handle_stripe(sh);
  4258. release_stripe(sh);
  4259. handled++;
  4260. }
  4261. remaining = raid5_dec_bi_active_stripes(raid_bio);
  4262. if (remaining == 0)
  4263. bio_endio(raid_bio, 0);
  4264. if (atomic_dec_and_test(&conf->active_aligned_reads))
  4265. wake_up(&conf->wait_for_stripe);
  4266. return handled;
  4267. }
  4268. #define MAX_STRIPE_BATCH 8
  4269. static int handle_active_stripes(struct r5conf *conf)
  4270. {
  4271. struct stripe_head *batch[MAX_STRIPE_BATCH], *sh;
  4272. int i, batch_size = 0;
  4273. while (batch_size < MAX_STRIPE_BATCH &&
  4274. (sh = __get_priority_stripe(conf)) != NULL)
  4275. batch[batch_size++] = sh;
  4276. if (batch_size == 0)
  4277. return batch_size;
  4278. spin_unlock_irq(&conf->device_lock);
  4279. for (i = 0; i < batch_size; i++)
  4280. handle_stripe(batch[i]);
  4281. cond_resched();
  4282. spin_lock_irq(&conf->device_lock);
  4283. for (i = 0; i < batch_size; i++)
  4284. __release_stripe(conf, batch[i]);
  4285. return batch_size;
  4286. }
  4287. /*
  4288. * This is our raid5 kernel thread.
  4289. *
  4290. * We scan the hash table for stripes which can be handled now.
  4291. * During the scan, completed stripes are saved for us by the interrupt
  4292. * handler, so that they will not have to wait for our next wakeup.
  4293. */
  4294. static void raid5d(struct md_thread *thread)
  4295. {
  4296. struct mddev *mddev = thread->mddev;
  4297. struct r5conf *conf = mddev->private;
  4298. int handled;
  4299. struct blk_plug plug;
  4300. pr_debug("+++ raid5d active\n");
  4301. md_check_recovery(mddev);
  4302. blk_start_plug(&plug);
  4303. handled = 0;
  4304. spin_lock_irq(&conf->device_lock);
  4305. while (1) {
  4306. struct bio *bio;
  4307. int batch_size;
  4308. if (
  4309. !list_empty(&conf->bitmap_list)) {
  4310. /* Now is a good time to flush some bitmap updates */
  4311. conf->seq_flush++;
  4312. spin_unlock_irq(&conf->device_lock);
  4313. bitmap_unplug(mddev->bitmap);
  4314. spin_lock_irq(&conf->device_lock);
  4315. conf->seq_write = conf->seq_flush;
  4316. activate_bit_delay(conf);
  4317. }
  4318. raid5_activate_delayed(conf);
  4319. while ((bio = remove_bio_from_retry(conf))) {
  4320. int ok;
  4321. spin_unlock_irq(&conf->device_lock);
  4322. ok = retry_aligned_read(conf, bio);
  4323. spin_lock_irq(&conf->device_lock);
  4324. if (!ok)
  4325. break;
  4326. handled++;
  4327. }
  4328. batch_size = handle_active_stripes(conf);
  4329. if (!batch_size)
  4330. break;
  4331. handled += batch_size;
  4332. if (mddev->flags & ~(1<<MD_CHANGE_PENDING)) {
  4333. spin_unlock_irq(&conf->device_lock);
  4334. md_check_recovery(mddev);
  4335. spin_lock_irq(&conf->device_lock);
  4336. }
  4337. }
  4338. pr_debug("%d stripes handled\n", handled);
  4339. spin_unlock_irq(&conf->device_lock);
  4340. async_tx_issue_pending_all();
  4341. blk_finish_plug(&plug);
  4342. pr_debug("--- raid5d inactive\n");
  4343. }
  4344. static ssize_t
  4345. raid5_show_stripe_cache_size(struct mddev *mddev, char *page)
  4346. {
  4347. struct r5conf *conf = mddev->private;
  4348. if (conf)
  4349. return sprintf(page, "%d\n", conf->max_nr_stripes);
  4350. else
  4351. return 0;
  4352. }
  4353. int
  4354. raid5_set_cache_size(struct mddev *mddev, int size)
  4355. {
  4356. struct r5conf *conf = mddev->private;
  4357. int err;
  4358. if (size <= 16 || size > 32768)
  4359. return -EINVAL;
  4360. while (size < conf->max_nr_stripes) {
  4361. if (drop_one_stripe(conf))
  4362. conf->max_nr_stripes--;
  4363. else
  4364. break;
  4365. }
  4366. err = md_allow_write(mddev);
  4367. if (err)
  4368. return err;
  4369. while (size > conf->max_nr_stripes) {
  4370. if (grow_one_stripe(conf))
  4371. conf->max_nr_stripes++;
  4372. else break;
  4373. }
  4374. return 0;
  4375. }
  4376. EXPORT_SYMBOL(raid5_set_cache_size);
  4377. static ssize_t
  4378. raid5_store_stripe_cache_size(struct mddev *mddev, const char *page, size_t len)
  4379. {
  4380. struct r5conf *conf = mddev->private;
  4381. unsigned long new;
  4382. int err;
  4383. if (len >= PAGE_SIZE)
  4384. return -EINVAL;
  4385. if (!conf)
  4386. return -ENODEV;
  4387. if (strict_strtoul(page, 10, &new))
  4388. return -EINVAL;
  4389. err = raid5_set_cache_size(mddev, new);
  4390. if (err)
  4391. return err;
  4392. return len;
  4393. }
  4394. static struct md_sysfs_entry
  4395. raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR,
  4396. raid5_show_stripe_cache_size,
  4397. raid5_store_stripe_cache_size);
  4398. static ssize_t
  4399. raid5_show_preread_threshold(struct mddev *mddev, char *page)
  4400. {
  4401. struct r5conf *conf = mddev->private;
  4402. if (conf)
  4403. return sprintf(page, "%d\n", conf->bypass_threshold);
  4404. else
  4405. return 0;
  4406. }
  4407. static ssize_t
  4408. raid5_store_preread_threshold(struct mddev *mddev, const char *page, size_t len)
  4409. {
  4410. struct r5conf *conf = mddev->private;
  4411. unsigned long new;
  4412. if (len >= PAGE_SIZE)
  4413. return -EINVAL;
  4414. if (!conf)
  4415. return -ENODEV;
  4416. if (strict_strtoul(page, 10, &new))
  4417. return -EINVAL;
  4418. if (new > conf->max_nr_stripes)
  4419. return -EINVAL;
  4420. conf->bypass_threshold = new;
  4421. return len;
  4422. }
  4423. static struct md_sysfs_entry
  4424. raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold,
  4425. S_IRUGO | S_IWUSR,
  4426. raid5_show_preread_threshold,
  4427. raid5_store_preread_threshold);
  4428. static ssize_t
  4429. stripe_cache_active_show(struct mddev *mddev, char *page)
  4430. {
  4431. struct r5conf *conf = mddev->private;
  4432. if (conf)
  4433. return sprintf(page, "%d\n", atomic_read(&conf->active_stripes));
  4434. else
  4435. return 0;
  4436. }
  4437. static struct md_sysfs_entry
  4438. raid5_stripecache_active = __ATTR_RO(stripe_cache_active);
  4439. static struct attribute *raid5_attrs[] = {
  4440. &raid5_stripecache_size.attr,
  4441. &raid5_stripecache_active.attr,
  4442. &raid5_preread_bypass_threshold.attr,
  4443. NULL,
  4444. };
  4445. static struct attribute_group raid5_attrs_group = {
  4446. .name = NULL,
  4447. .attrs = raid5_attrs,
  4448. };
  4449. static sector_t
  4450. raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks)
  4451. {
  4452. struct r5conf *conf = mddev->private;
  4453. if (!sectors)
  4454. sectors = mddev->dev_sectors;
  4455. if (!raid_disks)
  4456. /* size is defined by the smallest of previous and new size */
  4457. raid_disks = min(conf->raid_disks, conf->previous_raid_disks);
  4458. sectors &= ~((sector_t)mddev->chunk_sectors - 1);
  4459. sectors &= ~((sector_t)mddev->new_chunk_sectors - 1);
  4460. return sectors * (raid_disks - conf->max_degraded);
  4461. }
  4462. static void raid5_free_percpu(struct r5conf *conf)
  4463. {
  4464. struct raid5_percpu *percpu;
  4465. unsigned long cpu;
  4466. if (!conf->percpu)
  4467. return;
  4468. get_online_cpus();
  4469. for_each_possible_cpu(cpu) {
  4470. percpu = per_cpu_ptr(conf->percpu, cpu);
  4471. safe_put_page(percpu->spare_page);
  4472. kfree(percpu->scribble);
  4473. }
  4474. #ifdef CONFIG_HOTPLUG_CPU
  4475. unregister_cpu_notifier(&conf->cpu_notify);
  4476. #endif
  4477. put_online_cpus();
  4478. free_percpu(conf->percpu);
  4479. }
  4480. static void free_conf(struct r5conf *conf)
  4481. {
  4482. shrink_stripes(conf);
  4483. raid5_free_percpu(conf);
  4484. kfree(conf->disks);
  4485. kfree(conf->stripe_hashtbl);
  4486. kfree(conf);
  4487. }
  4488. #ifdef CONFIG_HOTPLUG_CPU
  4489. static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action,
  4490. void *hcpu)
  4491. {
  4492. struct r5conf *conf = container_of(nfb, struct r5conf, cpu_notify);
  4493. long cpu = (long)hcpu;
  4494. struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu);
  4495. switch (action) {
  4496. case CPU_UP_PREPARE:
  4497. case CPU_UP_PREPARE_FROZEN:
  4498. if (conf->level == 6 && !percpu->spare_page)
  4499. percpu->spare_page = alloc_page(GFP_KERNEL);
  4500. if (!percpu->scribble)
  4501. percpu->scribble = kmalloc(conf->scribble_len, GFP_KERNEL);
  4502. if (!percpu->scribble ||
  4503. (conf->level == 6 && !percpu->spare_page)) {
  4504. safe_put_page(percpu->spare_page);
  4505. kfree(percpu->scribble);
  4506. pr_err("%s: failed memory allocation for cpu%ld\n",
  4507. __func__, cpu);
  4508. return notifier_from_errno(-ENOMEM);
  4509. }
  4510. break;
  4511. case CPU_DEAD:
  4512. case CPU_DEAD_FROZEN:
  4513. safe_put_page(percpu->spare_page);
  4514. kfree(percpu->scribble);
  4515. percpu->spare_page = NULL;
  4516. percpu->scribble = NULL;
  4517. break;
  4518. default:
  4519. break;
  4520. }
  4521. return NOTIFY_OK;
  4522. }
  4523. #endif
  4524. static int raid5_alloc_percpu(struct r5conf *conf)
  4525. {
  4526. unsigned long cpu;
  4527. struct page *spare_page;
  4528. struct raid5_percpu __percpu *allcpus;
  4529. void *scribble;
  4530. int err;
  4531. allcpus = alloc_percpu(struct raid5_percpu);
  4532. if (!allcpus)
  4533. return -ENOMEM;
  4534. conf->percpu = allcpus;
  4535. get_online_cpus();
  4536. err = 0;
  4537. for_each_present_cpu(cpu) {
  4538. if (conf->level == 6) {
  4539. spare_page = alloc_page(GFP_KERNEL);
  4540. if (!spare_page) {
  4541. err = -ENOMEM;
  4542. break;
  4543. }
  4544. per_cpu_ptr(conf->percpu, cpu)->spare_page = spare_page;
  4545. }
  4546. scribble = kmalloc(conf->scribble_len, GFP_KERNEL);
  4547. if (!scribble) {
  4548. err = -ENOMEM;
  4549. break;
  4550. }
  4551. per_cpu_ptr(conf->percpu, cpu)->scribble = scribble;
  4552. }
  4553. #ifdef CONFIG_HOTPLUG_CPU
  4554. conf->cpu_notify.notifier_call = raid456_cpu_notify;
  4555. conf->cpu_notify.priority = 0;
  4556. if (err == 0)
  4557. err = register_cpu_notifier(&conf->cpu_notify);
  4558. #endif
  4559. put_online_cpus();
  4560. return err;
  4561. }
  4562. static struct r5conf *setup_conf(struct mddev *mddev)
  4563. {
  4564. struct r5conf *conf;
  4565. int raid_disk, memory, max_disks;
  4566. struct md_rdev *rdev;
  4567. struct disk_info *disk;
  4568. char pers_name[6];
  4569. if (mddev->new_level != 5
  4570. && mddev->new_level != 4
  4571. && mddev->new_level != 6) {
  4572. printk(KERN_ERR "md/raid:%s: raid level not set to 4/5/6 (%d)\n",
  4573. mdname(mddev), mddev->new_level);
  4574. return ERR_PTR(-EIO);
  4575. }
  4576. if ((mddev->new_level == 5
  4577. && !algorithm_valid_raid5(mddev->new_layout)) ||
  4578. (mddev->new_level == 6
  4579. && !algorithm_valid_raid6(mddev->new_layout))) {
  4580. printk(KERN_ERR "md/raid:%s: layout %d not supported\n",
  4581. mdname(mddev), mddev->new_layout);
  4582. return ERR_PTR(-EIO);
  4583. }
  4584. if (mddev->new_level == 6 && mddev->raid_disks < 4) {
  4585. printk(KERN_ERR "md/raid:%s: not enough configured devices (%d, minimum 4)\n",
  4586. mdname(mddev), mddev->raid_disks);
  4587. return ERR_PTR(-EINVAL);
  4588. }
  4589. if (!mddev->new_chunk_sectors ||
  4590. (mddev->new_chunk_sectors << 9) % PAGE_SIZE ||
  4591. !is_power_of_2(mddev->new_chunk_sectors)) {
  4592. printk(KERN_ERR "md/raid:%s: invalid chunk size %d\n",
  4593. mdname(mddev), mddev->new_chunk_sectors << 9);
  4594. return ERR_PTR(-EINVAL);
  4595. }
  4596. conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL);
  4597. if (conf == NULL)
  4598. goto abort;
  4599. spin_lock_init(&conf->device_lock);
  4600. init_waitqueue_head(&conf->wait_for_stripe);
  4601. init_waitqueue_head(&conf->wait_for_overlap);
  4602. INIT_LIST_HEAD(&conf->handle_list);
  4603. INIT_LIST_HEAD(&conf->hold_list);
  4604. INIT_LIST_HEAD(&conf->delayed_list);
  4605. INIT_LIST_HEAD(&conf->bitmap_list);
  4606. INIT_LIST_HEAD(&conf->inactive_list);
  4607. atomic_set(&conf->active_stripes, 0);
  4608. atomic_set(&conf->preread_active_stripes, 0);
  4609. atomic_set(&conf->active_aligned_reads, 0);
  4610. conf->bypass_threshold = BYPASS_THRESHOLD;
  4611. conf->recovery_disabled = mddev->recovery_disabled - 1;
  4612. conf->raid_disks = mddev->raid_disks;
  4613. if (mddev->reshape_position == MaxSector)
  4614. conf->previous_raid_disks = mddev->raid_disks;
  4615. else
  4616. conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks;
  4617. max_disks = max(conf->raid_disks, conf->previous_raid_disks);
  4618. conf->scribble_len = scribble_len(max_disks);
  4619. conf->disks = kzalloc(max_disks * sizeof(struct disk_info),
  4620. GFP_KERNEL);
  4621. if (!conf->disks)
  4622. goto abort;
  4623. conf->mddev = mddev;
  4624. if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL)
  4625. goto abort;
  4626. conf->level = mddev->new_level;
  4627. if (raid5_alloc_percpu(conf) != 0)
  4628. goto abort;
  4629. pr_debug("raid456: run(%s) called.\n", mdname(mddev));
  4630. rdev_for_each(rdev, mddev) {
  4631. raid_disk = rdev->raid_disk;
  4632. if (raid_disk >= max_disks
  4633. || raid_disk < 0)
  4634. continue;
  4635. disk = conf->disks + raid_disk;
  4636. if (test_bit(Replacement, &rdev->flags)) {
  4637. if (disk->replacement)
  4638. goto abort;
  4639. disk->replacement = rdev;
  4640. } else {
  4641. if (disk->rdev)
  4642. goto abort;
  4643. disk->rdev = rdev;
  4644. }
  4645. if (test_bit(In_sync, &rdev->flags)) {
  4646. char b[BDEVNAME_SIZE];
  4647. printk(KERN_INFO "md/raid:%s: device %s operational as raid"
  4648. " disk %d\n",
  4649. mdname(mddev), bdevname(rdev->bdev, b), raid_disk);
  4650. } else if (rdev->saved_raid_disk != raid_disk)
  4651. /* Cannot rely on bitmap to complete recovery */
  4652. conf->fullsync = 1;
  4653. }
  4654. conf->chunk_sectors = mddev->new_chunk_sectors;
  4655. conf->level = mddev->new_level;
  4656. if (conf->level == 6)
  4657. conf->max_degraded = 2;
  4658. else
  4659. conf->max_degraded = 1;
  4660. conf->algorithm = mddev->new_layout;
  4661. conf->max_nr_stripes = NR_STRIPES;
  4662. conf->reshape_progress = mddev->reshape_position;
  4663. if (conf->reshape_progress != MaxSector) {
  4664. conf->prev_chunk_sectors = mddev->chunk_sectors;
  4665. conf->prev_algo = mddev->layout;
  4666. }
  4667. memory = conf->max_nr_stripes * (sizeof(struct stripe_head) +
  4668. max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024;
  4669. if (grow_stripes(conf, conf->max_nr_stripes)) {
  4670. printk(KERN_ERR
  4671. "md/raid:%s: couldn't allocate %dkB for buffers\n",
  4672. mdname(mddev), memory);
  4673. goto abort;
  4674. } else
  4675. printk(KERN_INFO "md/raid:%s: allocated %dkB\n",
  4676. mdname(mddev), memory);
  4677. sprintf(pers_name, "raid%d", mddev->new_level);
  4678. conf->thread = md_register_thread(raid5d, mddev, pers_name);
  4679. if (!conf->thread) {
  4680. printk(KERN_ERR
  4681. "md/raid:%s: couldn't allocate thread.\n",
  4682. mdname(mddev));
  4683. goto abort;
  4684. }
  4685. return conf;
  4686. abort:
  4687. if (conf) {
  4688. free_conf(conf);
  4689. return ERR_PTR(-EIO);
  4690. } else
  4691. return ERR_PTR(-ENOMEM);
  4692. }
  4693. static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded)
  4694. {
  4695. switch (algo) {
  4696. case ALGORITHM_PARITY_0:
  4697. if (raid_disk < max_degraded)
  4698. return 1;
  4699. break;
  4700. case ALGORITHM_PARITY_N:
  4701. if (raid_disk >= raid_disks - max_degraded)
  4702. return 1;
  4703. break;
  4704. case ALGORITHM_PARITY_0_6:
  4705. if (raid_disk == 0 ||
  4706. raid_disk == raid_disks - 1)
  4707. return 1;
  4708. break;
  4709. case ALGORITHM_LEFT_ASYMMETRIC_6:
  4710. case ALGORITHM_RIGHT_ASYMMETRIC_6:
  4711. case ALGORITHM_LEFT_SYMMETRIC_6:
  4712. case ALGORITHM_RIGHT_SYMMETRIC_6:
  4713. if (raid_disk == raid_disks - 1)
  4714. return 1;
  4715. }
  4716. return 0;
  4717. }
  4718. static int run(struct mddev *mddev)
  4719. {
  4720. struct r5conf *conf;
  4721. int working_disks = 0;
  4722. int dirty_parity_disks = 0;
  4723. struct md_rdev *rdev;
  4724. sector_t reshape_offset = 0;
  4725. int i;
  4726. long long min_offset_diff = 0;
  4727. int first = 1;
  4728. if (mddev->recovery_cp != MaxSector)
  4729. printk(KERN_NOTICE "md/raid:%s: not clean"
  4730. " -- starting background reconstruction\n",
  4731. mdname(mddev));
  4732. rdev_for_each(rdev, mddev) {
  4733. long long diff;
  4734. if (rdev->raid_disk < 0)
  4735. continue;
  4736. diff = (rdev->new_data_offset - rdev->data_offset);
  4737. if (first) {
  4738. min_offset_diff = diff;
  4739. first = 0;
  4740. } else if (mddev->reshape_backwards &&
  4741. diff < min_offset_diff)
  4742. min_offset_diff = diff;
  4743. else if (!mddev->reshape_backwards &&
  4744. diff > min_offset_diff)
  4745. min_offset_diff = diff;
  4746. }
  4747. if (mddev->reshape_position != MaxSector) {
  4748. /* Check that we can continue the reshape.
  4749. * Difficulties arise if the stripe we would write to
  4750. * next is at or after the stripe we would read from next.
  4751. * For a reshape that changes the number of devices, this
  4752. * is only possible for a very short time, and mdadm makes
  4753. * sure that time appears to have past before assembling
  4754. * the array. So we fail if that time hasn't passed.
  4755. * For a reshape that keeps the number of devices the same
  4756. * mdadm must be monitoring the reshape can keeping the
  4757. * critical areas read-only and backed up. It will start
  4758. * the array in read-only mode, so we check for that.
  4759. */
  4760. sector_t here_new, here_old;
  4761. int old_disks;
  4762. int max_degraded = (mddev->level == 6 ? 2 : 1);
  4763. if (mddev->new_level != mddev->level) {
  4764. printk(KERN_ERR "md/raid:%s: unsupported reshape "
  4765. "required - aborting.\n",
  4766. mdname(mddev));
  4767. return -EINVAL;
  4768. }
  4769. old_disks = mddev->raid_disks - mddev->delta_disks;
  4770. /* reshape_position must be on a new-stripe boundary, and one
  4771. * further up in new geometry must map after here in old
  4772. * geometry.
  4773. */
  4774. here_new = mddev->reshape_position;
  4775. if (sector_div(here_new, mddev->new_chunk_sectors *
  4776. (mddev->raid_disks - max_degraded))) {
  4777. printk(KERN_ERR "md/raid:%s: reshape_position not "
  4778. "on a stripe boundary\n", mdname(mddev));
  4779. return -EINVAL;
  4780. }
  4781. reshape_offset = here_new * mddev->new_chunk_sectors;
  4782. /* here_new is the stripe we will write to */
  4783. here_old = mddev->reshape_position;
  4784. sector_div(here_old, mddev->chunk_sectors *
  4785. (old_disks-max_degraded));
  4786. /* here_old is the first stripe that we might need to read
  4787. * from */
  4788. if (mddev->delta_disks == 0) {
  4789. if ((here_new * mddev->new_chunk_sectors !=
  4790. here_old * mddev->chunk_sectors)) {
  4791. printk(KERN_ERR "md/raid:%s: reshape position is"
  4792. " confused - aborting\n", mdname(mddev));
  4793. return -EINVAL;
  4794. }
  4795. /* We cannot be sure it is safe to start an in-place
  4796. * reshape. It is only safe if user-space is monitoring
  4797. * and taking constant backups.
  4798. * mdadm always starts a situation like this in
  4799. * readonly mode so it can take control before
  4800. * allowing any writes. So just check for that.
  4801. */
  4802. if (abs(min_offset_diff) >= mddev->chunk_sectors &&
  4803. abs(min_offset_diff) >= mddev->new_chunk_sectors)
  4804. /* not really in-place - so OK */;
  4805. else if (mddev->ro == 0) {
  4806. printk(KERN_ERR "md/raid:%s: in-place reshape "
  4807. "must be started in read-only mode "
  4808. "- aborting\n",
  4809. mdname(mddev));
  4810. return -EINVAL;
  4811. }
  4812. } else if (mddev->reshape_backwards
  4813. ? (here_new * mddev->new_chunk_sectors + min_offset_diff <=
  4814. here_old * mddev->chunk_sectors)
  4815. : (here_new * mddev->new_chunk_sectors >=
  4816. here_old * mddev->chunk_sectors + (-min_offset_diff))) {
  4817. /* Reading from the same stripe as writing to - bad */
  4818. printk(KERN_ERR "md/raid:%s: reshape_position too early for "
  4819. "auto-recovery - aborting.\n",
  4820. mdname(mddev));
  4821. return -EINVAL;
  4822. }
  4823. printk(KERN_INFO "md/raid:%s: reshape will continue\n",
  4824. mdname(mddev));
  4825. /* OK, we should be able to continue; */
  4826. } else {
  4827. BUG_ON(mddev->level != mddev->new_level);
  4828. BUG_ON(mddev->layout != mddev->new_layout);
  4829. BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors);
  4830. BUG_ON(mddev->delta_disks != 0);
  4831. }
  4832. if (mddev->private == NULL)
  4833. conf = setup_conf(mddev);
  4834. else
  4835. conf = mddev->private;
  4836. if (IS_ERR(conf))
  4837. return PTR_ERR(conf);
  4838. conf->min_offset_diff = min_offset_diff;
  4839. mddev->thread = conf->thread;
  4840. conf->thread = NULL;
  4841. mddev->private = conf;
  4842. for (i = 0; i < conf->raid_disks && conf->previous_raid_disks;
  4843. i++) {
  4844. rdev = conf->disks[i].rdev;
  4845. if (!rdev && conf->disks[i].replacement) {
  4846. /* The replacement is all we have yet */
  4847. rdev = conf->disks[i].replacement;
  4848. conf->disks[i].replacement = NULL;
  4849. clear_bit(Replacement, &rdev->flags);
  4850. conf->disks[i].rdev = rdev;
  4851. }
  4852. if (!rdev)
  4853. continue;
  4854. if (conf->disks[i].replacement &&
  4855. conf->reshape_progress != MaxSector) {
  4856. /* replacements and reshape simply do not mix. */
  4857. printk(KERN_ERR "md: cannot handle concurrent "
  4858. "replacement and reshape.\n");
  4859. goto abort;
  4860. }
  4861. if (test_bit(In_sync, &rdev->flags)) {
  4862. working_disks++;
  4863. continue;
  4864. }
  4865. /* This disc is not fully in-sync. However if it
  4866. * just stored parity (beyond the recovery_offset),
  4867. * when we don't need to be concerned about the
  4868. * array being dirty.
  4869. * When reshape goes 'backwards', we never have
  4870. * partially completed devices, so we only need
  4871. * to worry about reshape going forwards.
  4872. */
  4873. /* Hack because v0.91 doesn't store recovery_offset properly. */
  4874. if (mddev->major_version == 0 &&
  4875. mddev->minor_version > 90)
  4876. rdev->recovery_offset = reshape_offset;
  4877. if (rdev->recovery_offset < reshape_offset) {
  4878. /* We need to check old and new layout */
  4879. if (!only_parity(rdev->raid_disk,
  4880. conf->algorithm,
  4881. conf->raid_disks,
  4882. conf->max_degraded))
  4883. continue;
  4884. }
  4885. if (!only_parity(rdev->raid_disk,
  4886. conf->prev_algo,
  4887. conf->previous_raid_disks,
  4888. conf->max_degraded))
  4889. continue;
  4890. dirty_parity_disks++;
  4891. }
  4892. /*
  4893. * 0 for a fully functional array, 1 or 2 for a degraded array.
  4894. */
  4895. mddev->degraded = calc_degraded(conf);
  4896. if (has_failed(conf)) {
  4897. printk(KERN_ERR "md/raid:%s: not enough operational devices"
  4898. " (%d/%d failed)\n",
  4899. mdname(mddev), mddev->degraded, conf->raid_disks);
  4900. goto abort;
  4901. }
  4902. /* device size must be a multiple of chunk size */
  4903. mddev->dev_sectors &= ~(mddev->chunk_sectors - 1);
  4904. mddev->resync_max_sectors = mddev->dev_sectors;
  4905. if (mddev->degraded > dirty_parity_disks &&
  4906. mddev->recovery_cp != MaxSector) {
  4907. if (mddev->ok_start_degraded)
  4908. printk(KERN_WARNING
  4909. "md/raid:%s: starting dirty degraded array"
  4910. " - data corruption possible.\n",
  4911. mdname(mddev));
  4912. else {
  4913. printk(KERN_ERR
  4914. "md/raid:%s: cannot start dirty degraded array.\n",
  4915. mdname(mddev));
  4916. goto abort;
  4917. }
  4918. }
  4919. if (mddev->degraded == 0)
  4920. printk(KERN_INFO "md/raid:%s: raid level %d active with %d out of %d"
  4921. " devices, algorithm %d\n", mdname(mddev), conf->level,
  4922. mddev->raid_disks-mddev->degraded, mddev->raid_disks,
  4923. mddev->new_layout);
  4924. else
  4925. printk(KERN_ALERT "md/raid:%s: raid level %d active with %d"
  4926. " out of %d devices, algorithm %d\n",
  4927. mdname(mddev), conf->level,
  4928. mddev->raid_disks - mddev->degraded,
  4929. mddev->raid_disks, mddev->new_layout);
  4930. print_raid5_conf(conf);
  4931. if (conf->reshape_progress != MaxSector) {
  4932. conf->reshape_safe = conf->reshape_progress;
  4933. atomic_set(&conf->reshape_stripes, 0);
  4934. clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
  4935. clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
  4936. set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
  4937. set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
  4938. mddev->sync_thread = md_register_thread(md_do_sync, mddev,
  4939. "reshape");
  4940. }
  4941. /* Ok, everything is just fine now */
  4942. if (mddev->to_remove == &raid5_attrs_group)
  4943. mddev->to_remove = NULL;
  4944. else if (mddev->kobj.sd &&
  4945. sysfs_create_group(&mddev->kobj, &raid5_attrs_group))
  4946. printk(KERN_WARNING
  4947. "raid5: failed to create sysfs attributes for %s\n",
  4948. mdname(mddev));
  4949. md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
  4950. if (mddev->queue) {
  4951. int chunk_size;
  4952. bool discard_supported = true;
  4953. /* read-ahead size must cover two whole stripes, which
  4954. * is 2 * (datadisks) * chunksize where 'n' is the
  4955. * number of raid devices
  4956. */
  4957. int data_disks = conf->previous_raid_disks - conf->max_degraded;
  4958. int stripe = data_disks *
  4959. ((mddev->chunk_sectors << 9) / PAGE_SIZE);
  4960. if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
  4961. mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
  4962. blk_queue_merge_bvec(mddev->queue, raid5_mergeable_bvec);
  4963. mddev->queue->backing_dev_info.congested_data = mddev;
  4964. mddev->queue->backing_dev_info.congested_fn = raid5_congested;
  4965. chunk_size = mddev->chunk_sectors << 9;
  4966. blk_queue_io_min(mddev->queue, chunk_size);
  4967. blk_queue_io_opt(mddev->queue, chunk_size *
  4968. (conf->raid_disks - conf->max_degraded));
  4969. /*
  4970. * We can only discard a whole stripe. It doesn't make sense to
  4971. * discard data disk but write parity disk
  4972. */
  4973. stripe = stripe * PAGE_SIZE;
  4974. /* Round up to power of 2, as discard handling
  4975. * currently assumes that */
  4976. while ((stripe-1) & stripe)
  4977. stripe = (stripe | (stripe-1)) + 1;
  4978. mddev->queue->limits.discard_alignment = stripe;
  4979. mddev->queue->limits.discard_granularity = stripe;
  4980. /*
  4981. * unaligned part of discard request will be ignored, so can't
  4982. * guarantee discard_zerors_data
  4983. */
  4984. mddev->queue->limits.discard_zeroes_data = 0;
  4985. rdev_for_each(rdev, mddev) {
  4986. disk_stack_limits(mddev->gendisk, rdev->bdev,
  4987. rdev->data_offset << 9);
  4988. disk_stack_limits(mddev->gendisk, rdev->bdev,
  4989. rdev->new_data_offset << 9);
  4990. /*
  4991. * discard_zeroes_data is required, otherwise data
  4992. * could be lost. Consider a scenario: discard a stripe
  4993. * (the stripe could be inconsistent if
  4994. * discard_zeroes_data is 0); write one disk of the
  4995. * stripe (the stripe could be inconsistent again
  4996. * depending on which disks are used to calculate
  4997. * parity); the disk is broken; The stripe data of this
  4998. * disk is lost.
  4999. */
  5000. if (!blk_queue_discard(bdev_get_queue(rdev->bdev)) ||
  5001. !bdev_get_queue(rdev->bdev)->
  5002. limits.discard_zeroes_data)
  5003. discard_supported = false;
  5004. }
  5005. if (discard_supported &&
  5006. mddev->queue->limits.max_discard_sectors >= stripe &&
  5007. mddev->queue->limits.discard_granularity >= stripe)
  5008. queue_flag_set_unlocked(QUEUE_FLAG_DISCARD,
  5009. mddev->queue);
  5010. else
  5011. queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD,
  5012. mddev->queue);
  5013. }
  5014. return 0;
  5015. abort:
  5016. md_unregister_thread(&mddev->thread);
  5017. print_raid5_conf(conf);
  5018. free_conf(conf);
  5019. mddev->private = NULL;
  5020. printk(KERN_ALERT "md/raid:%s: failed to run raid set.\n", mdname(mddev));
  5021. return -EIO;
  5022. }
  5023. static int stop(struct mddev *mddev)
  5024. {
  5025. struct r5conf *conf = mddev->private;
  5026. md_unregister_thread(&mddev->thread);
  5027. if (mddev->queue)
  5028. mddev->queue->backing_dev_info.congested_fn = NULL;
  5029. free_conf(conf);
  5030. mddev->private = NULL;
  5031. mddev->to_remove = &raid5_attrs_group;
  5032. return 0;
  5033. }
  5034. static void status(struct seq_file *seq, struct mddev *mddev)
  5035. {
  5036. struct r5conf *conf = mddev->private;
  5037. int i;
  5038. seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level,
  5039. mddev->chunk_sectors / 2, mddev->layout);
  5040. seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded);
  5041. for (i = 0; i < conf->raid_disks; i++)
  5042. seq_printf (seq, "%s",
  5043. conf->disks[i].rdev &&
  5044. test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_");
  5045. seq_printf (seq, "]");
  5046. }
  5047. static void print_raid5_conf (struct r5conf *conf)
  5048. {
  5049. int i;
  5050. struct disk_info *tmp;
  5051. printk(KERN_DEBUG "RAID conf printout:\n");
  5052. if (!conf) {
  5053. printk("(conf==NULL)\n");
  5054. return;
  5055. }
  5056. printk(KERN_DEBUG " --- level:%d rd:%d wd:%d\n", conf->level,
  5057. conf->raid_disks,
  5058. conf->raid_disks - conf->mddev->degraded);
  5059. for (i = 0; i < conf->raid_disks; i++) {
  5060. char b[BDEVNAME_SIZE];
  5061. tmp = conf->disks + i;
  5062. if (tmp->rdev)
  5063. printk(KERN_DEBUG " disk %d, o:%d, dev:%s\n",
  5064. i, !test_bit(Faulty, &tmp->rdev->flags),
  5065. bdevname(tmp->rdev->bdev, b));
  5066. }
  5067. }
  5068. static int raid5_spare_active(struct mddev *mddev)
  5069. {
  5070. int i;
  5071. struct r5conf *conf = mddev->private;
  5072. struct disk_info *tmp;
  5073. int count = 0;
  5074. unsigned long flags;
  5075. for (i = 0; i < conf->raid_disks; i++) {
  5076. tmp = conf->disks + i;
  5077. if (tmp->replacement
  5078. && tmp->replacement->recovery_offset == MaxSector
  5079. && !test_bit(Faulty, &tmp->replacement->flags)
  5080. && !test_and_set_bit(In_sync, &tmp->replacement->flags)) {
  5081. /* Replacement has just become active. */
  5082. if (!tmp->rdev
  5083. || !test_and_clear_bit(In_sync, &tmp->rdev->flags))
  5084. count++;
  5085. if (tmp->rdev) {
  5086. /* Replaced device not technically faulty,
  5087. * but we need to be sure it gets removed
  5088. * and never re-added.
  5089. */
  5090. set_bit(Faulty, &tmp->rdev->flags);
  5091. sysfs_notify_dirent_safe(
  5092. tmp->rdev->sysfs_state);
  5093. }
  5094. sysfs_notify_dirent_safe(tmp->replacement->sysfs_state);
  5095. } else if (tmp->rdev
  5096. && tmp->rdev->recovery_offset == MaxSector
  5097. && !test_bit(Faulty, &tmp->rdev->flags)
  5098. && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
  5099. count++;
  5100. sysfs_notify_dirent_safe(tmp->rdev->sysfs_state);
  5101. }
  5102. }
  5103. spin_lock_irqsave(&conf->device_lock, flags);
  5104. mddev->degraded = calc_degraded(conf);
  5105. spin_unlock_irqrestore(&conf->device_lock, flags);
  5106. print_raid5_conf(conf);
  5107. return count;
  5108. }
  5109. static int raid5_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
  5110. {
  5111. struct r5conf *conf = mddev->private;
  5112. int err = 0;
  5113. int number = rdev->raid_disk;
  5114. struct md_rdev **rdevp;
  5115. struct disk_info *p = conf->disks + number;
  5116. print_raid5_conf(conf);
  5117. if (rdev == p->rdev)
  5118. rdevp = &p->rdev;
  5119. else if (rdev == p->replacement)
  5120. rdevp = &p->replacement;
  5121. else
  5122. return 0;
  5123. if (number >= conf->raid_disks &&
  5124. conf->reshape_progress == MaxSector)
  5125. clear_bit(In_sync, &rdev->flags);
  5126. if (test_bit(In_sync, &rdev->flags) ||
  5127. atomic_read(&rdev->nr_pending)) {
  5128. err = -EBUSY;
  5129. goto abort;
  5130. }
  5131. /* Only remove non-faulty devices if recovery
  5132. * isn't possible.
  5133. */
  5134. if (!test_bit(Faulty, &rdev->flags) &&
  5135. mddev->recovery_disabled != conf->recovery_disabled &&
  5136. !has_failed(conf) &&
  5137. (!p->replacement || p->replacement == rdev) &&
  5138. number < conf->raid_disks) {
  5139. err = -EBUSY;
  5140. goto abort;
  5141. }
  5142. *rdevp = NULL;
  5143. synchronize_rcu();
  5144. if (atomic_read(&rdev->nr_pending)) {
  5145. /* lost the race, try later */
  5146. err = -EBUSY;
  5147. *rdevp = rdev;
  5148. } else if (p->replacement) {
  5149. /* We must have just cleared 'rdev' */
  5150. p->rdev = p->replacement;
  5151. clear_bit(Replacement, &p->replacement->flags);
  5152. smp_mb(); /* Make sure other CPUs may see both as identical
  5153. * but will never see neither - if they are careful
  5154. */
  5155. p->replacement = NULL;
  5156. clear_bit(WantReplacement, &rdev->flags);
  5157. } else
  5158. /* We might have just removed the Replacement as faulty-
  5159. * clear the bit just in case
  5160. */
  5161. clear_bit(WantReplacement, &rdev->flags);
  5162. abort:
  5163. print_raid5_conf(conf);
  5164. return err;
  5165. }
  5166. static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev)
  5167. {
  5168. struct r5conf *conf = mddev->private;
  5169. int err = -EEXIST;
  5170. int disk;
  5171. struct disk_info *p;
  5172. int first = 0;
  5173. int last = conf->raid_disks - 1;
  5174. if (mddev->recovery_disabled == conf->recovery_disabled)
  5175. return -EBUSY;
  5176. if (rdev->saved_raid_disk < 0 && has_failed(conf))
  5177. /* no point adding a device */
  5178. return -EINVAL;
  5179. if (rdev->raid_disk >= 0)
  5180. first = last = rdev->raid_disk;
  5181. /*
  5182. * find the disk ... but prefer rdev->saved_raid_disk
  5183. * if possible.
  5184. */
  5185. if (rdev->saved_raid_disk >= 0 &&
  5186. rdev->saved_raid_disk >= first &&
  5187. conf->disks[rdev->saved_raid_disk].rdev == NULL)
  5188. first = rdev->saved_raid_disk;
  5189. for (disk = first; disk <= last; disk++) {
  5190. p = conf->disks + disk;
  5191. if (p->rdev == NULL) {
  5192. clear_bit(In_sync, &rdev->flags);
  5193. rdev->raid_disk = disk;
  5194. err = 0;
  5195. if (rdev->saved_raid_disk != disk)
  5196. conf->fullsync = 1;
  5197. rcu_assign_pointer(p->rdev, rdev);
  5198. goto out;
  5199. }
  5200. }
  5201. for (disk = first; disk <= last; disk++) {
  5202. p = conf->disks + disk;
  5203. if (test_bit(WantReplacement, &p->rdev->flags) &&
  5204. p->replacement == NULL) {
  5205. clear_bit(In_sync, &rdev->flags);
  5206. set_bit(Replacement, &rdev->flags);
  5207. rdev->raid_disk = disk;
  5208. err = 0;
  5209. conf->fullsync = 1;
  5210. rcu_assign_pointer(p->replacement, rdev);
  5211. break;
  5212. }
  5213. }
  5214. out:
  5215. print_raid5_conf(conf);
  5216. return err;
  5217. }
  5218. static int raid5_resize(struct mddev *mddev, sector_t sectors)
  5219. {
  5220. /* no resync is happening, and there is enough space
  5221. * on all devices, so we can resize.
  5222. * We need to make sure resync covers any new space.
  5223. * If the array is shrinking we should possibly wait until
  5224. * any io in the removed space completes, but it hardly seems
  5225. * worth it.
  5226. */
  5227. sector_t newsize;
  5228. sectors &= ~((sector_t)mddev->chunk_sectors - 1);
  5229. newsize = raid5_size(mddev, sectors, mddev->raid_disks);
  5230. if (mddev->external_size &&
  5231. mddev->array_sectors > newsize)
  5232. return -EINVAL;
  5233. if (mddev->bitmap) {
  5234. int ret = bitmap_resize(mddev->bitmap, sectors, 0, 0);
  5235. if (ret)
  5236. return ret;
  5237. }
  5238. md_set_array_sectors(mddev, newsize);
  5239. set_capacity(mddev->gendisk, mddev->array_sectors);
  5240. revalidate_disk(mddev->gendisk);
  5241. if (sectors > mddev->dev_sectors &&
  5242. mddev->recovery_cp > mddev->dev_sectors) {
  5243. mddev->recovery_cp = mddev->dev_sectors;
  5244. set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
  5245. }
  5246. mddev->dev_sectors = sectors;
  5247. mddev->resync_max_sectors = sectors;
  5248. return 0;
  5249. }
  5250. static int check_stripe_cache(struct mddev *mddev)
  5251. {
  5252. /* Can only proceed if there are plenty of stripe_heads.
  5253. * We need a minimum of one full stripe,, and for sensible progress
  5254. * it is best to have about 4 times that.
  5255. * If we require 4 times, then the default 256 4K stripe_heads will
  5256. * allow for chunk sizes up to 256K, which is probably OK.
  5257. * If the chunk size is greater, user-space should request more
  5258. * stripe_heads first.
  5259. */
  5260. struct r5conf *conf = mddev->private;
  5261. if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4
  5262. > conf->max_nr_stripes ||
  5263. ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4
  5264. > conf->max_nr_stripes) {
  5265. printk(KERN_WARNING "md/raid:%s: reshape: not enough stripes. Needed %lu\n",
  5266. mdname(mddev),
  5267. ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9)
  5268. / STRIPE_SIZE)*4);
  5269. return 0;
  5270. }
  5271. return 1;
  5272. }
  5273. static int check_reshape(struct mddev *mddev)
  5274. {
  5275. struct r5conf *conf = mddev->private;
  5276. if (mddev->delta_disks == 0 &&
  5277. mddev->new_layout == mddev->layout &&
  5278. mddev->new_chunk_sectors == mddev->chunk_sectors)
  5279. return 0; /* nothing to do */
  5280. if (has_failed(conf))
  5281. return -EINVAL;
  5282. if (mddev->delta_disks < 0) {
  5283. /* We might be able to shrink, but the devices must
  5284. * be made bigger first.
  5285. * For raid6, 4 is the minimum size.
  5286. * Otherwise 2 is the minimum
  5287. */
  5288. int min = 2;
  5289. if (mddev->level == 6)
  5290. min = 4;
  5291. if (mddev->raid_disks + mddev->delta_disks < min)
  5292. return -EINVAL;
  5293. }
  5294. if (!check_stripe_cache(mddev))
  5295. return -ENOSPC;
  5296. return resize_stripes(conf, (conf->previous_raid_disks
  5297. + mddev->delta_disks));
  5298. }
  5299. static int raid5_start_reshape(struct mddev *mddev)
  5300. {
  5301. struct r5conf *conf = mddev->private;
  5302. struct md_rdev *rdev;
  5303. int spares = 0;
  5304. unsigned long flags;
  5305. if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
  5306. return -EBUSY;
  5307. if (!check_stripe_cache(mddev))
  5308. return -ENOSPC;
  5309. if (has_failed(conf))
  5310. return -EINVAL;
  5311. rdev_for_each(rdev, mddev) {
  5312. if (!test_bit(In_sync, &rdev->flags)
  5313. && !test_bit(Faulty, &rdev->flags))
  5314. spares++;
  5315. }
  5316. if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded)
  5317. /* Not enough devices even to make a degraded array
  5318. * of that size
  5319. */
  5320. return -EINVAL;
  5321. /* Refuse to reduce size of the array. Any reductions in
  5322. * array size must be through explicit setting of array_size
  5323. * attribute.
  5324. */
  5325. if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks)
  5326. < mddev->array_sectors) {
  5327. printk(KERN_ERR "md/raid:%s: array size must be reduced "
  5328. "before number of disks\n", mdname(mddev));
  5329. return -EINVAL;
  5330. }
  5331. atomic_set(&conf->reshape_stripes, 0);
  5332. spin_lock_irq(&conf->device_lock);
  5333. conf->previous_raid_disks = conf->raid_disks;
  5334. conf->raid_disks += mddev->delta_disks;
  5335. conf->prev_chunk_sectors = conf->chunk_sectors;
  5336. conf->chunk_sectors = mddev->new_chunk_sectors;
  5337. conf->prev_algo = conf->algorithm;
  5338. conf->algorithm = mddev->new_layout;
  5339. conf->generation++;
  5340. /* Code that selects data_offset needs to see the generation update
  5341. * if reshape_progress has been set - so a memory barrier needed.
  5342. */
  5343. smp_mb();
  5344. if (mddev->reshape_backwards)
  5345. conf->reshape_progress = raid5_size(mddev, 0, 0);
  5346. else
  5347. conf->reshape_progress = 0;
  5348. conf->reshape_safe = conf->reshape_progress;
  5349. spin_unlock_irq(&conf->device_lock);
  5350. /* Add some new drives, as many as will fit.
  5351. * We know there are enough to make the newly sized array work.
  5352. * Don't add devices if we are reducing the number of
  5353. * devices in the array. This is because it is not possible
  5354. * to correctly record the "partially reconstructed" state of
  5355. * such devices during the reshape and confusion could result.
  5356. */
  5357. if (mddev->delta_disks >= 0) {
  5358. rdev_for_each(rdev, mddev)
  5359. if (rdev->raid_disk < 0 &&
  5360. !test_bit(Faulty, &rdev->flags)) {
  5361. if (raid5_add_disk(mddev, rdev) == 0) {
  5362. if (rdev->raid_disk
  5363. >= conf->previous_raid_disks)
  5364. set_bit(In_sync, &rdev->flags);
  5365. else
  5366. rdev->recovery_offset = 0;
  5367. if (sysfs_link_rdev(mddev, rdev))
  5368. /* Failure here is OK */;
  5369. }
  5370. } else if (rdev->raid_disk >= conf->previous_raid_disks
  5371. && !test_bit(Faulty, &rdev->flags)) {
  5372. /* This is a spare that was manually added */
  5373. set_bit(In_sync, &rdev->flags);
  5374. }
  5375. /* When a reshape changes the number of devices,
  5376. * ->degraded is measured against the larger of the
  5377. * pre and post number of devices.
  5378. */
  5379. spin_lock_irqsave(&conf->device_lock, flags);
  5380. mddev->degraded = calc_degraded(conf);
  5381. spin_unlock_irqrestore(&conf->device_lock, flags);
  5382. }
  5383. mddev->raid_disks = conf->raid_disks;
  5384. mddev->reshape_position = conf->reshape_progress;
  5385. set_bit(MD_CHANGE_DEVS, &mddev->flags);
  5386. clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
  5387. clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
  5388. set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
  5389. set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
  5390. mddev->sync_thread = md_register_thread(md_do_sync, mddev,
  5391. "reshape");
  5392. if (!mddev->sync_thread) {
  5393. mddev->recovery = 0;
  5394. spin_lock_irq(&conf->device_lock);
  5395. mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks;
  5396. rdev_for_each(rdev, mddev)
  5397. rdev->new_data_offset = rdev->data_offset;
  5398. smp_wmb();
  5399. conf->reshape_progress = MaxSector;
  5400. mddev->reshape_position = MaxSector;
  5401. spin_unlock_irq(&conf->device_lock);
  5402. return -EAGAIN;
  5403. }
  5404. conf->reshape_checkpoint = jiffies;
  5405. md_wakeup_thread(mddev->sync_thread);
  5406. md_new_event(mddev);
  5407. return 0;
  5408. }
  5409. /* This is called from the reshape thread and should make any
  5410. * changes needed in 'conf'
  5411. */
  5412. static void end_reshape(struct r5conf *conf)
  5413. {
  5414. if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) {
  5415. struct md_rdev *rdev;
  5416. spin_lock_irq(&conf->device_lock);
  5417. conf->previous_raid_disks = conf->raid_disks;
  5418. rdev_for_each(rdev, conf->mddev)
  5419. rdev->data_offset = rdev->new_data_offset;
  5420. smp_wmb();
  5421. conf->reshape_progress = MaxSector;
  5422. spin_unlock_irq(&conf->device_lock);
  5423. wake_up(&conf->wait_for_overlap);
  5424. /* read-ahead size must cover two whole stripes, which is
  5425. * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
  5426. */
  5427. if (conf->mddev->queue) {
  5428. int data_disks = conf->raid_disks - conf->max_degraded;
  5429. int stripe = data_disks * ((conf->chunk_sectors << 9)
  5430. / PAGE_SIZE);
  5431. if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
  5432. conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
  5433. }
  5434. }
  5435. }
  5436. /* This is called from the raid5d thread with mddev_lock held.
  5437. * It makes config changes to the device.
  5438. */
  5439. static void raid5_finish_reshape(struct mddev *mddev)
  5440. {
  5441. struct r5conf *conf = mddev->private;
  5442. if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
  5443. if (mddev->delta_disks > 0) {
  5444. md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
  5445. set_capacity(mddev->gendisk, mddev->array_sectors);
  5446. revalidate_disk(mddev->gendisk);
  5447. } else {
  5448. int d;
  5449. spin_lock_irq(&conf->device_lock);
  5450. mddev->degraded = calc_degraded(conf);
  5451. spin_unlock_irq(&conf->device_lock);
  5452. for (d = conf->raid_disks ;
  5453. d < conf->raid_disks - mddev->delta_disks;
  5454. d++) {
  5455. struct md_rdev *rdev = conf->disks[d].rdev;
  5456. if (rdev)
  5457. clear_bit(In_sync, &rdev->flags);
  5458. rdev = conf->disks[d].replacement;
  5459. if (rdev)
  5460. clear_bit(In_sync, &rdev->flags);
  5461. }
  5462. }
  5463. mddev->layout = conf->algorithm;
  5464. mddev->chunk_sectors = conf->chunk_sectors;
  5465. mddev->reshape_position = MaxSector;
  5466. mddev->delta_disks = 0;
  5467. mddev->reshape_backwards = 0;
  5468. }
  5469. }
  5470. static void raid5_quiesce(struct mddev *mddev, int state)
  5471. {
  5472. struct r5conf *conf = mddev->private;
  5473. switch(state) {
  5474. case 2: /* resume for a suspend */
  5475. wake_up(&conf->wait_for_overlap);
  5476. break;
  5477. case 1: /* stop all writes */
  5478. spin_lock_irq(&conf->device_lock);
  5479. /* '2' tells resync/reshape to pause so that all
  5480. * active stripes can drain
  5481. */
  5482. conf->quiesce = 2;
  5483. wait_event_lock_irq(conf->wait_for_stripe,
  5484. atomic_read(&conf->active_stripes) == 0 &&
  5485. atomic_read(&conf->active_aligned_reads) == 0,
  5486. conf->device_lock);
  5487. conf->quiesce = 1;
  5488. spin_unlock_irq(&conf->device_lock);
  5489. /* allow reshape to continue */
  5490. wake_up(&conf->wait_for_overlap);
  5491. break;
  5492. case 0: /* re-enable writes */
  5493. spin_lock_irq(&conf->device_lock);
  5494. conf->quiesce = 0;
  5495. wake_up(&conf->wait_for_stripe);
  5496. wake_up(&conf->wait_for_overlap);
  5497. spin_unlock_irq(&conf->device_lock);
  5498. break;
  5499. }
  5500. }
  5501. static void *raid45_takeover_raid0(struct mddev *mddev, int level)
  5502. {
  5503. struct r0conf *raid0_conf = mddev->private;
  5504. sector_t sectors;
  5505. /* for raid0 takeover only one zone is supported */
  5506. if (raid0_conf->nr_strip_zones > 1) {
  5507. printk(KERN_ERR "md/raid:%s: cannot takeover raid0 with more than one zone.\n",
  5508. mdname(mddev));
  5509. return ERR_PTR(-EINVAL);
  5510. }
  5511. sectors = raid0_conf->strip_zone[0].zone_end;
  5512. sector_div(sectors, raid0_conf->strip_zone[0].nb_dev);
  5513. mddev->dev_sectors = sectors;
  5514. mddev->new_level = level;
  5515. mddev->new_layout = ALGORITHM_PARITY_N;
  5516. mddev->new_chunk_sectors = mddev->chunk_sectors;
  5517. mddev->raid_disks += 1;
  5518. mddev->delta_disks = 1;
  5519. /* make sure it will be not marked as dirty */
  5520. mddev->recovery_cp = MaxSector;
  5521. return setup_conf(mddev);
  5522. }
  5523. static void *raid5_takeover_raid1(struct mddev *mddev)
  5524. {
  5525. int chunksect;
  5526. if (mddev->raid_disks != 2 ||
  5527. mddev->degraded > 1)
  5528. return ERR_PTR(-EINVAL);
  5529. /* Should check if there are write-behind devices? */
  5530. chunksect = 64*2; /* 64K by default */
  5531. /* The array must be an exact multiple of chunksize */
  5532. while (chunksect && (mddev->array_sectors & (chunksect-1)))
  5533. chunksect >>= 1;
  5534. if ((chunksect<<9) < STRIPE_SIZE)
  5535. /* array size does not allow a suitable chunk size */
  5536. return ERR_PTR(-EINVAL);
  5537. mddev->new_level = 5;
  5538. mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC;
  5539. mddev->new_chunk_sectors = chunksect;
  5540. return setup_conf(mddev);
  5541. }
  5542. static void *raid5_takeover_raid6(struct mddev *mddev)
  5543. {
  5544. int new_layout;
  5545. switch (mddev->layout) {
  5546. case ALGORITHM_LEFT_ASYMMETRIC_6:
  5547. new_layout = ALGORITHM_LEFT_ASYMMETRIC;
  5548. break;
  5549. case ALGORITHM_RIGHT_ASYMMETRIC_6:
  5550. new_layout = ALGORITHM_RIGHT_ASYMMETRIC;
  5551. break;
  5552. case ALGORITHM_LEFT_SYMMETRIC_6:
  5553. new_layout = ALGORITHM_LEFT_SYMMETRIC;
  5554. break;
  5555. case ALGORITHM_RIGHT_SYMMETRIC_6:
  5556. new_layout = ALGORITHM_RIGHT_SYMMETRIC;
  5557. break;
  5558. case ALGORITHM_PARITY_0_6:
  5559. new_layout = ALGORITHM_PARITY_0;
  5560. break;
  5561. case ALGORITHM_PARITY_N:
  5562. new_layout = ALGORITHM_PARITY_N;
  5563. break;
  5564. default:
  5565. return ERR_PTR(-EINVAL);
  5566. }
  5567. mddev->new_level = 5;
  5568. mddev->new_layout = new_layout;
  5569. mddev->delta_disks = -1;
  5570. mddev->raid_disks -= 1;
  5571. return setup_conf(mddev);
  5572. }
  5573. static int raid5_check_reshape(struct mddev *mddev)
  5574. {
  5575. /* For a 2-drive array, the layout and chunk size can be changed
  5576. * immediately as not restriping is needed.
  5577. * For larger arrays we record the new value - after validation
  5578. * to be used by a reshape pass.
  5579. */
  5580. struct r5conf *conf = mddev->private;
  5581. int new_chunk = mddev->new_chunk_sectors;
  5582. if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout))
  5583. return -EINVAL;
  5584. if (new_chunk > 0) {
  5585. if (!is_power_of_2(new_chunk))
  5586. return -EINVAL;
  5587. if (new_chunk < (PAGE_SIZE>>9))
  5588. return -EINVAL;
  5589. if (mddev->array_sectors & (new_chunk-1))
  5590. /* not factor of array size */
  5591. return -EINVAL;
  5592. }
  5593. /* They look valid */
  5594. if (mddev->raid_disks == 2) {
  5595. /* can make the change immediately */
  5596. if (mddev->new_layout >= 0) {
  5597. conf->algorithm = mddev->new_layout;
  5598. mddev->layout = mddev->new_layout;
  5599. }
  5600. if (new_chunk > 0) {
  5601. conf->chunk_sectors = new_chunk ;
  5602. mddev->chunk_sectors = new_chunk;
  5603. }
  5604. set_bit(MD_CHANGE_DEVS, &mddev->flags);
  5605. md_wakeup_thread(mddev->thread);
  5606. }
  5607. return check_reshape(mddev);
  5608. }
  5609. static int raid6_check_reshape(struct mddev *mddev)
  5610. {
  5611. int new_chunk = mddev->new_chunk_sectors;
  5612. if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout))
  5613. return -EINVAL;
  5614. if (new_chunk > 0) {
  5615. if (!is_power_of_2(new_chunk))
  5616. return -EINVAL;
  5617. if (new_chunk < (PAGE_SIZE >> 9))
  5618. return -EINVAL;
  5619. if (mddev->array_sectors & (new_chunk-1))
  5620. /* not factor of array size */
  5621. return -EINVAL;
  5622. }
  5623. /* They look valid */
  5624. return check_reshape(mddev);
  5625. }
  5626. static void *raid5_takeover(struct mddev *mddev)
  5627. {
  5628. /* raid5 can take over:
  5629. * raid0 - if there is only one strip zone - make it a raid4 layout
  5630. * raid1 - if there are two drives. We need to know the chunk size
  5631. * raid4 - trivial - just use a raid4 layout.
  5632. * raid6 - Providing it is a *_6 layout
  5633. */
  5634. if (mddev->level == 0)
  5635. return raid45_takeover_raid0(mddev, 5);
  5636. if (mddev->level == 1)
  5637. return raid5_takeover_raid1(mddev);
  5638. if (mddev->level == 4) {
  5639. mddev->new_layout = ALGORITHM_PARITY_N;
  5640. mddev->new_level = 5;
  5641. return setup_conf(mddev);
  5642. }
  5643. if (mddev->level == 6)
  5644. return raid5_takeover_raid6(mddev);
  5645. return ERR_PTR(-EINVAL);
  5646. }
  5647. static void *raid4_takeover(struct mddev *mddev)
  5648. {
  5649. /* raid4 can take over:
  5650. * raid0 - if there is only one strip zone
  5651. * raid5 - if layout is right
  5652. */
  5653. if (mddev->level == 0)
  5654. return raid45_takeover_raid0(mddev, 4);
  5655. if (mddev->level == 5 &&
  5656. mddev->layout == ALGORITHM_PARITY_N) {
  5657. mddev->new_layout = 0;
  5658. mddev->new_level = 4;
  5659. return setup_conf(mddev);
  5660. }
  5661. return ERR_PTR(-EINVAL);
  5662. }
  5663. static struct md_personality raid5_personality;
  5664. static void *raid6_takeover(struct mddev *mddev)
  5665. {
  5666. /* Currently can only take over a raid5. We map the
  5667. * personality to an equivalent raid6 personality
  5668. * with the Q block at the end.
  5669. */
  5670. int new_layout;
  5671. if (mddev->pers != &raid5_personality)
  5672. return ERR_PTR(-EINVAL);
  5673. if (mddev->degraded > 1)
  5674. return ERR_PTR(-EINVAL);
  5675. if (mddev->raid_disks > 253)
  5676. return ERR_PTR(-EINVAL);
  5677. if (mddev->raid_disks < 3)
  5678. return ERR_PTR(-EINVAL);
  5679. switch (mddev->layout) {
  5680. case ALGORITHM_LEFT_ASYMMETRIC:
  5681. new_layout = ALGORITHM_LEFT_ASYMMETRIC_6;
  5682. break;
  5683. case ALGORITHM_RIGHT_ASYMMETRIC:
  5684. new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6;
  5685. break;
  5686. case ALGORITHM_LEFT_SYMMETRIC:
  5687. new_layout = ALGORITHM_LEFT_SYMMETRIC_6;
  5688. break;
  5689. case ALGORITHM_RIGHT_SYMMETRIC:
  5690. new_layout = ALGORITHM_RIGHT_SYMMETRIC_6;
  5691. break;
  5692. case ALGORITHM_PARITY_0:
  5693. new_layout = ALGORITHM_PARITY_0_6;
  5694. break;
  5695. case ALGORITHM_PARITY_N:
  5696. new_layout = ALGORITHM_PARITY_N;
  5697. break;
  5698. default:
  5699. return ERR_PTR(-EINVAL);
  5700. }
  5701. mddev->new_level = 6;
  5702. mddev->new_layout = new_layout;
  5703. mddev->delta_disks = 1;
  5704. mddev->raid_disks += 1;
  5705. return setup_conf(mddev);
  5706. }
  5707. static struct md_personality raid6_personality =
  5708. {
  5709. .name = "raid6",
  5710. .level = 6,
  5711. .owner = THIS_MODULE,
  5712. .make_request = make_request,
  5713. .run = run,
  5714. .stop = stop,
  5715. .status = status,
  5716. .error_handler = error,
  5717. .hot_add_disk = raid5_add_disk,
  5718. .hot_remove_disk= raid5_remove_disk,
  5719. .spare_active = raid5_spare_active,
  5720. .sync_request = sync_request,
  5721. .resize = raid5_resize,
  5722. .size = raid5_size,
  5723. .check_reshape = raid6_check_reshape,
  5724. .start_reshape = raid5_start_reshape,
  5725. .finish_reshape = raid5_finish_reshape,
  5726. .quiesce = raid5_quiesce,
  5727. .takeover = raid6_takeover,
  5728. };
  5729. static struct md_personality raid5_personality =
  5730. {
  5731. .name = "raid5",
  5732. .level = 5,
  5733. .owner = THIS_MODULE,
  5734. .make_request = make_request,
  5735. .run = run,
  5736. .stop = stop,
  5737. .status = status,
  5738. .error_handler = error,
  5739. .hot_add_disk = raid5_add_disk,
  5740. .hot_remove_disk= raid5_remove_disk,
  5741. .spare_active = raid5_spare_active,
  5742. .sync_request = sync_request,
  5743. .resize = raid5_resize,
  5744. .size = raid5_size,
  5745. .check_reshape = raid5_check_reshape,
  5746. .start_reshape = raid5_start_reshape,
  5747. .finish_reshape = raid5_finish_reshape,
  5748. .quiesce = raid5_quiesce,
  5749. .takeover = raid5_takeover,
  5750. };
  5751. static struct md_personality raid4_personality =
  5752. {
  5753. .name = "raid4",
  5754. .level = 4,
  5755. .owner = THIS_MODULE,
  5756. .make_request = make_request,
  5757. .run = run,
  5758. .stop = stop,
  5759. .status = status,
  5760. .error_handler = error,
  5761. .hot_add_disk = raid5_add_disk,
  5762. .hot_remove_disk= raid5_remove_disk,
  5763. .spare_active = raid5_spare_active,
  5764. .sync_request = sync_request,
  5765. .resize = raid5_resize,
  5766. .size = raid5_size,
  5767. .check_reshape = raid5_check_reshape,
  5768. .start_reshape = raid5_start_reshape,
  5769. .finish_reshape = raid5_finish_reshape,
  5770. .quiesce = raid5_quiesce,
  5771. .takeover = raid4_takeover,
  5772. };
  5773. static int __init raid5_init(void)
  5774. {
  5775. register_md_personality(&raid6_personality);
  5776. register_md_personality(&raid5_personality);
  5777. register_md_personality(&raid4_personality);
  5778. return 0;
  5779. }
  5780. static void raid5_exit(void)
  5781. {
  5782. unregister_md_personality(&raid6_personality);
  5783. unregister_md_personality(&raid5_personality);
  5784. unregister_md_personality(&raid4_personality);
  5785. }
  5786. module_init(raid5_init);
  5787. module_exit(raid5_exit);
  5788. MODULE_LICENSE("GPL");
  5789. MODULE_DESCRIPTION("RAID4/5/6 (striping with parity) personality for MD");
  5790. MODULE_ALIAS("md-personality-4"); /* RAID5 */
  5791. MODULE_ALIAS("md-raid5");
  5792. MODULE_ALIAS("md-raid4");
  5793. MODULE_ALIAS("md-level-5");
  5794. MODULE_ALIAS("md-level-4");
  5795. MODULE_ALIAS("md-personality-8"); /* RAID6 */
  5796. MODULE_ALIAS("md-raid6");
  5797. MODULE_ALIAS("md-level-6");
  5798. /* This used to be two separate modules, they were: */
  5799. MODULE_ALIAS("raid5");
  5800. MODULE_ALIAS("raid6");